Features:

Install:

<dependency>    <groupId>us.codecraft</groupId>    <artifactId>webmagic-core</artifactId>    <version>${webmagic.version}</version></dependency><dependency>    <groupId>us.codecraft</groupId>    <artifactId>webmagic-extension</artifactId>    <version>${webmagic.version}</version></dependency>

<exclusions>    <exclusion>        <groupId>org.slf4j</groupId>        <artifactId>slf4j-log4j12</artifactId>    </exclusion></exclusions>

Get Started:

First crawler:

publicclassGithubRepoPageProcessorimplementsPageProcessor {privateSitesite =Site.me().setRetryTimes(3).setSleepTime(1000);@Overridepublicvoidprocess(Pagepage) {page.addTargetRequests(page.getHtml().links().regex("(https://github\\.com/\\w+/\\w+)").all());page.putField("author",page.getUrl().regex("https://github\\.com/(\\w+)/.*").toString());page.putField("name",page.getHtml().xpath("//h1[@class='public']/strong/a/text()").toString());if (page.getResultItems().get("name")==null){//skip this pagepage.setSkip(true);        }page.putField("readme",page.getHtml().xpath("//div[@id='readme']/tidyText()"));    }@OverridepublicSitegetSite() {returnsite;    }publicstaticvoidmain(String[]args) {Spider.create(newGithubRepoPageProcessor()).addUrl("https://github.com/code4craft").thread(5).run();    }}

@TargetUrl("https://github.com/\\w+/\\w+")@HelpUrl("https://github.com/\\w+")publicclassGithubRepo {@ExtractBy(value ="//h1[@class='public']/strong/a/text()",notNull =true)privateStringname;@ExtractByUrl("https://github\\.com/(\\w+)/.*")privateStringauthor;@ExtractBy("//div[@id='readme']/tidyText()")privateStringreadme;publicstaticvoidmain(String[]args) {OOSpider.create(Site.me().setSleepTime(1000)                ,newConsolePageModelPipeline(),GithubRepo.class)                .addUrl("https://github.com/code4craft").thread(5).run();    }}

Docs and samples:

License:

Thanks:

Mail-list:

Related Project