<addkey="myget.org"value="https://www.myget.org/F/zlzforever/api/v3/index.json"protocolVersion="3"/>

DESIGN

DEVELOP ENVIROMENT

MORE DOCUMENTS

SAMPLES

Please see the Project DotnetSpider.Sample in the solution.

BASE USAGE

ADDITIONAL USAGE: Configurable Entity Spider

[DisplayName("博客园爬虫")]publicclassEntitySpider(IOptions<SpiderOptions>options,DependenceServicesservices,ILogger<Spider>logger):Spider(options,services,logger){publicstaticasyncTaskRunAsync(){varbuilder=Builder.CreateDefaultBuilder<EntitySpider>(options=>{options.Speed=1;});builder.UseSerilog();builder.IgnoreServerCertificateError();awaitbuilder.Build().RunAsync();}protectedoverrideasyncTaskInitializeAsync(CancellationTokenstoppingToken=default){AddDataFlow<DataParser<CnblogsEntry>>();AddDataFlow(GetDefaultStorage);awaitAddRequestsAsync(newRequest("https://news.cnblogs.com/n/page/1",newDictionary<string,object>{{"网站","博客园"}}));}[Schema("cnblogs","news")][EntitySelector(Expression=".//div[@class='news_block']",Type=SelectorType.XPath)][GlobalValueSelector(Expression=".//a[@class='current']",Name="类别",Type=SelectorType.XPath)][GlobalValueSelector(Expression="//title",Name="Title",Type=SelectorType.XPath)][FollowRequestSelector(Expressions=["//div[@class='pager']"])]publicclassCnblogsEntry:EntityBase<CnblogsEntry>{protectedoverridevoidConfigure(){HasIndex(x=>x.Title);HasIndex(x=>new{x.WebSite,x.Guid},true);}publicintId{get;set;}[Required][StringLength(200)][ValueSelector(Expression="类别",Type=SelectorType.Environment)]publicstringCategory{get;set;}[Required][StringLength(200)][ValueSelector(Expression="网站",Type=SelectorType.Environment)]publicstringWebSite{get;set;}[StringLength(200)][ValueSelector(Expression="Title",Type=SelectorType.Environment)][ReplaceFormatter(NewValue="",OldValue=" - 博客园")]publicstringTitle{get;set;}[StringLength(40)][ValueSelector(Expression="GUID",Type=SelectorType.Environment)]publicstringGuid{get;set;}[ValueSelector(Expression=".//h2[@class='news_entry']/a")]publicstringNews{get;set;}[ValueSelector(Expression=".//h2[@class='news_entry']/a/@href")]publicstringUrl{get;set;}[ValueSelector(Expression=".//div[@class='entry_summary']")][TrimFormatter]publicstringPlainText{get;set;}[ValueSelector(Expression="DATETIME",Type=SelectorType.Environment)]publicDateTimeCreationTime{get;set;}}}

Distributed spider

Puppeteer downloader

NOTICE

when you use redis scheduler, please update your redis config:

timeout 0tcp-keepalive 60

Dependencies

AREAS FOR IMPROVEMENTS