Commit776ed9c

committed

added link scraper

1 parent7409f60 commit776ed9cCopy full SHA for 776ed9c

File tree

+53

-8

lines changed

+53

-8

lines changed

Lines changed: 9 additions & 8 deletions

Original file line number	Diff line number	Diff line change
`@@ -6,6 +6,7 @@`
`6`	`6`	`email_re=re.compile(r'([\w\.,]+@[\w\.,]+\.\w+)')`
`7`	`7`	`link_re=re.compile(r'href="(.*?)"')`
`8`	`8`
	`9`	`+`
`9`	`10`	`defcrawl(url,maxlevel):`
`10`	`11`
`11`	`12`	`result=set()`
`@@ -25,21 +26,21 @@ def crawl(url, maxlevel):`
`25`	`26`	`# Get an absolute URL for a link`
`26`	`27`	`link=urlparse.urljoin(url,link)`
`27`	`28`
`28`		`-# Find all emails on current page`
`29`		`-result.update(email_re.findall(req.text))`
	`29`	`+# Find all emails on current page`
	`30`	`+result.update(email_re.findall(req.text))`
`30`	`31`
`31`		`-print"Crawled level: {}".format(maxlevel)`
	`32`	`+print"Crawled level: {}".format(maxlevel)`
`32`	`33`
`33`		`-# new level`
`34`		`-maxlevel-=1`
	`34`	`+# new level`
	`35`	`+maxlevel-=1`
`35`	`36`
`36`		`-# recurse`
`37`		`-crawl(link,maxlevel)`
	`37`	`+# recurse`
	`38`	`+crawl(link,maxlevel)`
`38`	`39`
`39`	`40`	`returnresult`
`40`	`41`
`41`	`42`	`emails=crawl('http://www.website_goes_here_dot_com',2)`
`42`	`43`
`43`	`44`	`print"\nScrapped e-mail addresses:"`
`44`	`45`	`foremailinemails:`
`45`		`-printemail`
	`46`	`+printemail`

Lines changed: 44 additions & 0 deletions

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,44 @@`
	`1`	`+importrequests`
	`2`	`+importre`
	`3`	`+importurlparse`
	`4`	`+`
	`5`	`+# regex`
	`6`	`+link_re=re.compile(r'href="(.*?)"')`
	`7`	`+`
	`8`	`+`
	`9`	`+defcrawl(url,maxlevel):`
	`10`	`+`
	`11`	`+result=set()`
	`12`	`+`
	`13`	`+whilemaxlevel>0:`
	`14`	`+`
	`15`	`+# Get the webpage`
	`16`	`+req=requests.get(url)`
	`17`	`+`
	`18`	`+# Check if successful`
	`19`	`+if(req.status_code!=200):`
	`20`	`+return []`
	`21`	`+`
	`22`	`+# Find and follow all the links`
	`23`	`+links=link_re.findall(req.text)`
	`24`	`+forlinkinlinks:`
	`25`	`+# Get an absolute URL for a link`
	`26`	`+link=urlparse.urljoin(url,link)`
	`27`	`+# add links to result set`
	`28`	`+result.update(link)`
	`29`	`+`
	`30`	`+print"Crawled level: {}".format(maxlevel)`
	`31`	`+`
	`32`	`+# new level`
	`33`	`+maxlevel-=1`
	`34`	`+`
	`35`	`+# recurse`
	`36`	`+crawl(link,maxlevel)`
	`37`	`+`
	`38`	`+returnresult`
	`39`	`+`
	`40`	`+emails=crawl('http://www.website_goes_here_dot_com',2)`
	`41`	`+`
	`42`	`+print"\nScrapped links:"`
	`43`	`+forlinkinlinks:`
	`44`	`+printlink`

Comments

(0)