Commit406ef1d

committed

edit link extractor tutorial

1 parent272ee58 commit406ef1dCopy full SHA for 406ef1d

File tree

-2

lines changed

-2

lines changed

Lines changed: 4 additions & 1 deletion

Original file line number	Diff line number	Diff line change
`@@ -9,6 +9,7 @@`
`9`	`9`	`GREEN=colorama.Fore.GREEN`
`10`	`10`	`GRAY=colorama.Fore.LIGHTBLACK_EX`
`11`	`11`	`RESET=colorama.Fore.RESET`
	`12`	`+YELLOW=colorama.Fore.YELLOW`
`12`	`13`
`13`	`14`	`# initialize the set of links (unique links)`
`14`	`15`	`internal_urls=set()`
`@@ -62,7 +63,7 @@ def get_all_website_links(url):`
`62`	`63`	`returnurls`
`63`	`64`
`64`	`65`
`65`		`-defcrawl(url,max_urls=50):`
	`66`	`+defcrawl(url,max_urls=30):`
`66`	`67`	`"""`
`67`	`68`	`Crawls a web page and extracts all links.`
`68`	`69`	You'll find all links in `external_urls` and `internal_urls` global set variables.
`@@ -71,6 +72,7 @@ def crawl(url, max_urls=50):`
`71`	`72`	`"""`
`72`	`73`	`globaltotal_urls_visited`
`73`	`74`	`total_urls_visited+=1`
	`75`	`+print(f"{YELLOW}[*] Crawling:{url}{RESET}")`
`74`	`76`	`links=get_all_website_links(url)`
`75`	`77`	`forlinkinlinks:`
`76`	`78`	`iftotal_urls_visited>max_urls:`
`@@ -93,6 +95,7 @@ def crawl(url, max_urls=50):`
`93`	`95`	`print("[+] Total Internal links:",len(internal_urls))`
`94`	`96`	`print("[+] Total External links:",len(external_urls))`
`95`	`97`	`print("[+] Total URLs:",len(external_urls)+len(internal_urls))`
	`98`	`+print("[+] Total crawled URLs:",max_urls)`
`96`	`99`
`97`	`100`	`domain_name=urlparse(url).netloc`
`98`	`101`

Lines changed: 4 additions & 1 deletion

Original file line number	Diff line number	Diff line change
`@@ -9,6 +9,7 @@`
`9`	`9`	`GREEN=colorama.Fore.GREEN`
`10`	`10`	`GRAY=colorama.Fore.LIGHTBLACK_EX`
`11`	`11`	`RESET=colorama.Fore.RESET`
	`12`	`+YELLOW=colorama.Fore.YELLOW`
`12`	`13`
`13`	`14`	`# initialize the set of links (unique links)`
`14`	`15`	`internal_urls=set()`
`@@ -71,7 +72,7 @@ def get_all_website_links(url):`
`71`	`72`	`returnurls`
`72`	`73`
`73`	`74`
`74`		`-defcrawl(url,max_urls=50):`
	`75`	`+defcrawl(url,max_urls=30):`
`75`	`76`	`"""`
`76`	`77`	`Crawls a web page and extracts all links.`
`77`	`78`	You'll find all links in `external_urls` and `internal_urls` global set variables.
`@@ -80,6 +81,7 @@ def crawl(url, max_urls=50):`
`80`	`81`	`"""`
`81`	`82`	`globaltotal_urls_visited`
`82`	`83`	`total_urls_visited+=1`
	`84`	`+print(f"{YELLOW}[*] Crawling:{url}{RESET}")`
`83`	`85`	`links=get_all_website_links(url)`
`84`	`86`	`forlinkinlinks:`
`85`	`87`	`iftotal_urls_visited>max_urls:`
`@@ -102,6 +104,7 @@ def crawl(url, max_urls=50):`
`102`	`104`	`print("[+] Total Internal links:",len(internal_urls))`
`103`	`105`	`print("[+] Total External links:",len(external_urls))`
`104`	`106`	`print("[+] Total URLs:",len(external_urls)+len(internal_urls))`
	`107`	`+print("[+] Total crawled URLs:",max_urls)`
`105`	`108`
`106`	`109`	`domain_name=urlparse(url).netloc`
`107`	`110`

Comments

(0)