link_re=re.compile(r'href="(.*?)"')

defcrawl(url,maxlevel):

defcrawl(url):

result=set()

req=requests.get(url)

if(req.status_code!=200):

return []

if(req.status_code!=200):

return []

links=link_re.findall(req.text)

links=link_re.findall(req.text)

forlinkinlinks:

link=urlparse.urljoin(url,link)

print"\nFound {} links".format(len(links))

print"Crawled level: {}".format(maxlevel)

link=urlparse.urljoin(url,link)

crawl(link,maxlevel)

result.update(email_re.findall(req.text))

emails=crawl('http://www.website_goes_here_dot_com',2)

if__name__=='__main__':

emails=crawl('http://www.realpython.com')

foremailinemails:

foremailinemails:

link_re=re.compile(r'href="(.*?)"')

defcrawl(url,maxlevel):

defcrawl(url):

result=set()

req=requests.get(url)

whilemaxlevel>0:

if(req.status_code!=200):

return []

req=requests.get(url)

links=link_re.findall(req.text)

if(req.status_code!=200):

return []

print"\nFound {} links".format(len(links))

links=link_re.findall(req.text)

forlinkinlinks:

link=urlparse.urljoin(url,link)

result.update(link)

forlinkinlinks:

print"Crawled level: {}".format(maxlevel)

link=urlparse.urljoin(url,link)

crawl(link,maxlevel)

emails=crawl('http://www.website_goes_here_dot_com',2)

forlinkinlinks:

if__name__=='__main__':

crawl('http://www.realpython.com')

1.**05_load_json_without_dupes.py**: load json, convert to dict, raise error if there is a duplicate key

1.**06_execution_time.py**: class used for timing execution of code

1.**07_benchmark_permissions_loading_django.py**: benchmark loading of permissions in Django

1.**08_basic_email_web_crawler.py**: web crawler for grabbing emails from a website recursively

1.**09_basic_link_web_crawler.py**: web crawler for grabbing links from a websiterecursively

1.**08_basic_email_web_crawler.py**: web crawler for grabbing emails from a website

1.**09_basic_link_web_crawler.py**: web crawler for grabbing links from a website

1.**10_find_files_recursively.py**: recursively grab files from a directory