pythoncode-tutorials/web-scraping/link-extractor/link_extractor.py at master · hulinhui-code/pythoncode-tutorials · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
importrequests
fromurllib.parseimporturlparse, urljoin
frombs4importBeautifulSoup
importcolorama

# init the colorama module
colorama.init()

GREEN=colorama.Fore.GREEN
GRAY=colorama.Fore.LIGHTBLACK_EX
RESET=colorama.Fore.RESET

# initialize the set of links (unique links)
internal_urls=set()
external_urls=set()

total_urls_visited=0


defis_valid(url):
"""
 Checks whether `url` is a valid URL.
 """
parsed=urlparse(url)
returnbool(parsed.netloc) andbool(parsed.scheme)


defget_all_website_links(url):
"""
 Returns all URLs that is found on `url` in which it belongs to the same website
 """
# all URLs of `url`
urls=set()
# domain name of the URL without the protocol
domain_name=urlparse(url).netloc
soup=BeautifulSoup(requests.get(url).content, "html.parser")
fora_taginsoup.findAll("a"):
href=a_tag.attrs.get("href")
ifhref==""orhrefisNone:
# href empty tag
continue
# join the URL if it's relative (not absolute link)
href=urljoin(url, href)
parsed_href=urlparse(href)
# remove URL GET parameters, URL fragments, etc.
href=parsed_href.scheme+"://"+parsed_href.netloc+parsed_href.path
ifnotis_valid(href):
# not a valid URL
continue
ifhrefininternal_urls:
# already in the set
continue
ifdomain_namenotinhref:
# external link
ifhrefnotinexternal_urls:
print(f"{GRAY}[!] External link: {href}{RESET}")
external_urls.add(href)
continue
print(f"{GREEN}[*] Internal link: {href}{RESET}")
urls.add(href)
internal_urls.add(href)
returnurls


defcrawl(url, max_urls=50):
"""
 Crawls a web page and extracts all links.
 You'll find all links in `external_urls` and `internal_urls` global set variables.
 params:
 max_urls (int): number of max urls to crawl, default is 30.
 """
globaltotal_urls_visited
total_urls_visited+=1
links=get_all_website_links(url)
forlinkinlinks:
iftotal_urls_visited>max_urls:
break
crawl(link, max_urls=max_urls)


if__name__=="__main__":
importargparse
parser=argparse.ArgumentParser(description="Link Extractor Tool with Python")
parser.add_argument("url", help="The URL to extract links from.")
parser.add_argument("-m", "--max-urls", help="Number of max URLs to crawl, default is 30.", default=30, type=int)

args=parser.parse_args()
url=args.url
max_urls=args.max_urls

crawl(url, max_urls=max_urls)

print("[+] Total Internal links:", len(internal_urls))
print("[+] Total External links:", len(external_urls))
print("[+] Total URLs:", len(external_urls) +len(internal_urls))

domain_name=urlparse(url).netloc

# save the internal links to a file
withopen(f"{domain_name}_internal_links.txt", "w") asf:
forinternal_linkininternal_urls:
print(internal_link.strip(), file=f)

# save the external links to a file
withopen(f"{domain_name}_external_links.txt", "w") asf:
forexternal_linkinexternal_urls:
print(external_link.strip(), file=f)

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
importrequests
fromurllib.parseimporturlparse, urljoin
frombs4importBeautifulSoup
importcolorama

# init the colorama module
colorama.init()

GREEN=colorama.Fore.GREEN
GRAY=colorama.Fore.LIGHTBLACK_EX
RESET=colorama.Fore.RESET

# initialize the set of links (unique links)
internal_urls=set()
external_urls=set()

total_urls_visited=0


defis_valid(url):
"""
 Checks whether `url` is a valid URL.
 """
parsed=urlparse(url)
returnbool(parsed.netloc) andbool(parsed.scheme)


defget_all_website_links(url):
"""
 Returns all URLs that is found on `url` in which it belongs to the same website
 """
# all URLs of `url`
urls=set()
# domain name of the URL without the protocol
domain_name=urlparse(url).netloc
soup=BeautifulSoup(requests.get(url).content, "html.parser")
fora_taginsoup.findAll("a"):
href=a_tag.attrs.get("href")
ifhref==""orhrefisNone:
# href empty tag
continue
# join the URL if it's relative (not absolute link)
href=urljoin(url, href)
parsed_href=urlparse(href)
# remove URL GET parameters, URL fragments, etc.
href=parsed_href.scheme+"://"+parsed_href.netloc+parsed_href.path
ifnotis_valid(href):
# not a valid URL
continue
ifhrefininternal_urls:
# already in the set
continue
ifdomain_namenotinhref:
# external link
ifhrefnotinexternal_urls:
print(f"{GRAY}[!] External link: {href}{RESET}")
external_urls.add(href)
continue
print(f"{GREEN}[*] Internal link: {href}{RESET}")
urls.add(href)
internal_urls.add(href)
returnurls


defcrawl(url, max_urls=50):
"""
 Crawls a web page and extracts all links.
 You'll find all links in `external_urls` and `internal_urls` global set variables.
 params:
 max_urls (int): number of max urls to crawl, default is 30.
 """
globaltotal_urls_visited
total_urls_visited+=1
links=get_all_website_links(url)
forlinkinlinks:
iftotal_urls_visited>max_urls:
break
crawl(link, max_urls=max_urls)


if__name__=="__main__":
importargparse
parser=argparse.ArgumentParser(description="Link Extractor Tool with Python")
parser.add_argument("url", help="The URL to extract links from.")
parser.add_argument("-m", "--max-urls", help="Number of max URLs to crawl, default is 30.", default=30, type=int)

args=parser.parse_args()
url=args.url
max_urls=args.max_urls

crawl(url, max_urls=max_urls)

print("[+] Total Internal links:", len(internal_urls))
print("[+] Total External links:", len(external_urls))
print("[+] Total URLs:", len(external_urls) +len(internal_urls))

domain_name=urlparse(url).netloc

# save the internal links to a file
withopen(f"{domain_name}_internal_links.txt", "w") asf:
forinternal_linkininternal_urls:
print(internal_link.strip(), file=f)

# save the external links to a file
withopen(f"{domain_name}_external_links.txt", "w") asf:
forexternal_linkinexternal_urls:
print(external_link.strip(), file=f)

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
importrequests
fromurllib.parseimporturlparse, urljoin
frombs4importBeautifulSoup
importcolorama

# init the colorama module
colorama.init()

GREEN=colorama.Fore.GREEN
GRAY=colorama.Fore.LIGHTBLACK_EX
RESET=colorama.Fore.RESET

# initialize the set of links (unique links)
internal_urls=set()
external_urls=set()

total_urls_visited=0


defis_valid(url):
"""
 Checks whether `url` is a valid URL.
 """
parsed=urlparse(url)
returnbool(parsed.netloc) andbool(parsed.scheme)


defget_all_website_links(url):
"""
 Returns all URLs that is found on `url` in which it belongs to the same website
 """
# all URLs of `url`
urls=set()
# domain name of the URL without the protocol
domain_name=urlparse(url).netloc
soup=BeautifulSoup(requests.get(url).content, "html.parser")
fora_taginsoup.findAll("a"):
href=a_tag.attrs.get("href")
ifhref==""orhrefisNone:
# href empty tag
continue
# join the URL if it's relative (not absolute link)
href=urljoin(url, href)
parsed_href=urlparse(href)
# remove URL GET parameters, URL fragments, etc.
href=parsed_href.scheme+"://"+parsed_href.netloc+parsed_href.path
ifnotis_valid(href):
# not a valid URL
continue
ifhrefininternal_urls:
# already in the set
continue
ifdomain_namenotinhref:
# external link
ifhrefnotinexternal_urls:
print(f"{GRAY}[!] External link: {href}{RESET}")
external_urls.add(href)
continue
print(f"{GREEN}[*] Internal link: {href}{RESET}")
urls.add(href)
internal_urls.add(href)
returnurls


defcrawl(url, max_urls=50):
"""
 Crawls a web page and extracts all links.
 You'll find all links in `external_urls` and `internal_urls` global set variables.
 params:
 max_urls (int): number of max urls to crawl, default is 30.
 """
globaltotal_urls_visited
total_urls_visited+=1
links=get_all_website_links(url)
forlinkinlinks:
iftotal_urls_visited>max_urls:
break
crawl(link, max_urls=max_urls)


if__name__=="__main__":
importargparse
parser=argparse.ArgumentParser(description="Link Extractor Tool with Python")
parser.add_argument("url", help="The URL to extract links from.")
parser.add_argument("-m", "--max-urls", help="Number of max URLs to crawl, default is 30.", default=30, type=int)

args=parser.parse_args()
url=args.url
max_urls=args.max_urls

crawl(url, max_urls=max_urls)

print("[+] Total Internal links:", len(internal_urls))
print("[+] Total External links:", len(external_urls))
print("[+] Total URLs:", len(external_urls) +len(internal_urls))

domain_name=urlparse(url).netloc

# save the internal links to a file
withopen(f"{domain_name}_internal_links.txt", "w") asf:
forinternal_linkininternal_urls:
print(internal_link.strip(), file=f)

# save the external links to a file
withopen(f"{domain_name}_external_links.txt", "w") asf:
forexternal_linkinexternal_urls:
print(external_link.strip(), file=f)

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
importrequests
fromurllib.parseimporturlparse, urljoin
frombs4importBeautifulSoup
importcolorama

# init the colorama module
colorama.init()

GREEN=colorama.Fore.GREEN
GRAY=colorama.Fore.LIGHTBLACK_EX
RESET=colorama.Fore.RESET

# initialize the set of links (unique links)
internal_urls=set()
external_urls=set()

total_urls_visited=0


defis_valid(url):
"""
 Checks whether `url` is a valid URL.
 """
parsed=urlparse(url)
returnbool(parsed.netloc) andbool(parsed.scheme)


defget_all_website_links(url):
"""
 Returns all URLs that is found on `url` in which it belongs to the same website
 """
# all URLs of `url`
urls=set()
# domain name of the URL without the protocol
domain_name=urlparse(url).netloc
soup=BeautifulSoup(requests.get(url).content, "html.parser")
fora_taginsoup.findAll("a"):
href=a_tag.attrs.get("href")
ifhref==""orhrefisNone:
# href empty tag
continue
# join the URL if it's relative (not absolute link)
href=urljoin(url, href)
parsed_href=urlparse(href)
# remove URL GET parameters, URL fragments, etc.
href=parsed_href.scheme+"://"+parsed_href.netloc+parsed_href.path
ifnotis_valid(href):
# not a valid URL
continue
ifhrefininternal_urls:
# already in the set
continue
ifdomain_namenotinhref:
# external link
ifhrefnotinexternal_urls:
print(f"{GRAY}[!] External link: {href}{RESET}")
external_urls.add(href)
continue
print(f"{GREEN}[*] Internal link: {href}{RESET}")
urls.add(href)
internal_urls.add(href)
returnurls


defcrawl(url, max_urls=50):
"""
 Crawls a web page and extracts all links.
 You'll find all links in `external_urls` and `internal_urls` global set variables.
 params:
 max_urls (int): number of max urls to crawl, default is 30.
 """
globaltotal_urls_visited
total_urls_visited+=1
links=get_all_website_links(url)
forlinkinlinks:
iftotal_urls_visited>max_urls:
break
crawl(link, max_urls=max_urls)


if__name__=="__main__":
importargparse
parser=argparse.ArgumentParser(description="Link Extractor Tool with Python")
parser.add_argument("url", help="The URL to extract links from.")
parser.add_argument("-m", "--max-urls", help="Number of max URLs to crawl, default is 30.", default=30, type=int)

args=parser.parse_args()
url=args.url
max_urls=args.max_urls

crawl(url, max_urls=max_urls)

print("[+] Total Internal links:", len(internal_urls))
print("[+] Total External links:", len(external_urls))
print("[+] Total URLs:", len(external_urls) +len(internal_urls))

domain_name=urlparse(url).netloc

# save the internal links to a file
withopen(f"{domain_name}_internal_links.txt", "w") asf:
forinternal_linkininternal_urls:
print(internal_link.strip(), file=f)

# save the external links to a file
withopen(f"{domain_name}_external_links.txt", "w") asf:
forexternal_linkinexternal_urls:
print(external_link.strip(), file=f)

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
importrequests
fromurllib.parseimporturlparse, urljoin
frombs4importBeautifulSoup
importcolorama

# init the colorama module
colorama.init()

GREEN=colorama.Fore.GREEN
GRAY=colorama.Fore.LIGHTBLACK_EX
RESET=colorama.Fore.RESET

# initialize the set of links (unique links)
internal_urls=set()
external_urls=set()

total_urls_visited=0


defis_valid(url):
"""
 Checks whether `url` is a valid URL.
 """
parsed=urlparse(url)
returnbool(parsed.netloc) andbool(parsed.scheme)


defget_all_website_links(url):
"""
 Returns all URLs that is found on `url` in which it belongs to the same website
 """
# all URLs of `url`
urls=set()
# domain name of the URL without the protocol
domain_name=urlparse(url).netloc
soup=BeautifulSoup(requests.get(url).content, "html.parser")
fora_taginsoup.findAll("a"):
href=a_tag.attrs.get("href")
ifhref==""orhrefisNone:
# href empty tag
continue
# join the URL if it's relative (not absolute link)
href=urljoin(url, href)
parsed_href=urlparse(href)
# remove URL GET parameters, URL fragments, etc.
href=parsed_href.scheme+"://"+parsed_href.netloc+parsed_href.path
ifnotis_valid(href):
# not a valid URL
continue
ifhrefininternal_urls:
# already in the set
continue
ifdomain_namenotinhref:
# external link
ifhrefnotinexternal_urls:
print(f"{GRAY}[!] External link: {href}{RESET}")
external_urls.add(href)
continue
print(f"{GREEN}[*] Internal link: {href}{RESET}")
urls.add(href)
internal_urls.add(href)
returnurls


defcrawl(url, max_urls=50):
"""
 Crawls a web page and extracts all links.
 You'll find all links in `external_urls` and `internal_urls` global set variables.
 params:
 max_urls (int): number of max urls to crawl, default is 30.
 """
globaltotal_urls_visited
total_urls_visited+=1
links=get_all_website_links(url)
forlinkinlinks:
iftotal_urls_visited>max_urls:
break
crawl(link, max_urls=max_urls)


if__name__=="__main__":
importargparse
parser=argparse.ArgumentParser(description="Link Extractor Tool with Python")
parser.add_argument("url", help="The URL to extract links from.")
parser.add_argument("-m", "--max-urls", help="Number of max URLs to crawl, default is 30.", default=30, type=int)

args=parser.parse_args()
url=args.url
max_urls=args.max_urls

crawl(url, max_urls=max_urls)

print("[+] Total Internal links:", len(internal_urls))
print("[+] Total External links:", len(external_urls))
print("[+] Total URLs:", len(external_urls) +len(internal_urls))

domain_name=urlparse(url).netloc

# save the internal links to a file
withopen(f"{domain_name}_internal_links.txt", "w") asf:
forinternal_linkininternal_urls:
print(internal_link.strip(), file=f)

# save the external links to a file
withopen(f"{domain_name}_external_links.txt", "w") asf:
forexternal_linkinexternal_urls:
print(external_link.strip(), file=f)

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
importrequests
fromurllib.parseimporturlparse, urljoin
frombs4importBeautifulSoup
importcolorama

# init the colorama module
colorama.init()

GREEN=colorama.Fore.GREEN
GRAY=colorama.Fore.LIGHTBLACK_EX
RESET=colorama.Fore.RESET

# initialize the set of links (unique links)
internal_urls=set()
external_urls=set()

total_urls_visited=0


defis_valid(url):
"""
 Checks whether `url` is a valid URL.
 """
parsed=urlparse(url)
returnbool(parsed.netloc) andbool(parsed.scheme)


defget_all_website_links(url):
"""
 Returns all URLs that is found on `url` in which it belongs to the same website
 """
# all URLs of `url`
urls=set()
# domain name of the URL without the protocol
domain_name=urlparse(url).netloc
soup=BeautifulSoup(requests.get(url).content, "html.parser")
fora_taginsoup.findAll("a"):
href=a_tag.attrs.get("href")
ifhref==""orhrefisNone:
# href empty tag
continue
# join the URL if it's relative (not absolute link)
href=urljoin(url, href)
parsed_href=urlparse(href)
# remove URL GET parameters, URL fragments, etc.
href=parsed_href.scheme+"://"+parsed_href.netloc+parsed_href.path
ifnotis_valid(href):
# not a valid URL
continue
ifhrefininternal_urls:
# already in the set
continue
ifdomain_namenotinhref:
# external link
ifhrefnotinexternal_urls:
print(f"{GRAY}[!] External link: {href}{RESET}")
external_urls.add(href)
continue
print(f"{GREEN}[*] Internal link: {href}{RESET}")
urls.add(href)
internal_urls.add(href)
returnurls


defcrawl(url, max_urls=50):
"""
 Crawls a web page and extracts all links.
 You'll find all links in `external_urls` and `internal_urls` global set variables.
 params:
 max_urls (int): number of max urls to crawl, default is 30.
 """
globaltotal_urls_visited
total_urls_visited+=1
links=get_all_website_links(url)
forlinkinlinks:
iftotal_urls_visited>max_urls:
break
crawl(link, max_urls=max_urls)


if__name__=="__main__":
importargparse
parser=argparse.ArgumentParser(description="Link Extractor Tool with Python")
parser.add_argument("url", help="The URL to extract links from.")
parser.add_argument("-m", "--max-urls", help="Number of max URLs to crawl, default is 30.", default=30, type=int)

args=parser.parse_args()
url=args.url
max_urls=args.max_urls

crawl(url, max_urls=max_urls)

print("[+] Total Internal links:", len(internal_urls))
print("[+] Total External links:", len(external_urls))
print("[+] Total URLs:", len(external_urls) +len(internal_urls))

domain_name=urlparse(url).netloc

# save the internal links to a file
withopen(f"{domain_name}_internal_links.txt", "w") asf:
forinternal_linkininternal_urls:
print(internal_link.strip(), file=f)

# save the external links to a file
withopen(f"{domain_name}_external_links.txt", "w") asf:
forexternal_linkinexternal_urls:
print(external_link.strip(), file=f)

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
importrequests
fromurllib.parseimporturlparse, urljoin
frombs4importBeautifulSoup
importcolorama

# init the colorama module
colorama.init()

GREEN=colorama.Fore.GREEN
GRAY=colorama.Fore.LIGHTBLACK_EX
RESET=colorama.Fore.RESET

# initialize the set of links (unique links)
internal_urls=set()
external_urls=set()

total_urls_visited=0


defis_valid(url):
"""
 Checks whether `url` is a valid URL.
 """
parsed=urlparse(url)
returnbool(parsed.netloc) andbool(parsed.scheme)


defget_all_website_links(url):
"""
 Returns all URLs that is found on `url` in which it belongs to the same website
 """
# all URLs of `url`
urls=set()
# domain name of the URL without the protocol
domain_name=urlparse(url).netloc
soup=BeautifulSoup(requests.get(url).content, "html.parser")
fora_taginsoup.findAll("a"):
href=a_tag.attrs.get("href")
ifhref==""orhrefisNone:
# href empty tag
continue
# join the URL if it's relative (not absolute link)
href=urljoin(url, href)
parsed_href=urlparse(href)
# remove URL GET parameters, URL fragments, etc.
href=parsed_href.scheme+"://"+parsed_href.netloc+parsed_href.path
ifnotis_valid(href):
# not a valid URL
continue
ifhrefininternal_urls:
# already in the set
continue
ifdomain_namenotinhref:
# external link
ifhrefnotinexternal_urls:
print(f"{GRAY}[!] External link: {href}{RESET}")
external_urls.add(href)
continue
print(f"{GREEN}[*] Internal link: {href}{RESET}")
urls.add(href)
internal_urls.add(href)
returnurls


defcrawl(url, max_urls=50):
"""
 Crawls a web page and extracts all links.
 You'll find all links in `external_urls` and `internal_urls` global set variables.
 params:
 max_urls (int): number of max urls to crawl, default is 30.
 """
globaltotal_urls_visited
total_urls_visited+=1
links=get_all_website_links(url)
forlinkinlinks:
iftotal_urls_visited>max_urls:
break
crawl(link, max_urls=max_urls)


if__name__=="__main__":
importargparse
parser=argparse.ArgumentParser(description="Link Extractor Tool with Python")
parser.add_argument("url", help="The URL to extract links from.")
parser.add_argument("-m", "--max-urls", help="Number of max URLs to crawl, default is 30.", default=30, type=int)

args=parser.parse_args()
url=args.url
max_urls=args.max_urls

crawl(url, max_urls=max_urls)

print("[+] Total Internal links:", len(internal_urls))
print("[+] Total External links:", len(external_urls))
print("[+] Total URLs:", len(external_urls) +len(internal_urls))

domain_name=urlparse(url).netloc

# save the internal links to a file
withopen(f"{domain_name}_internal_links.txt", "w") asf:
forinternal_linkininternal_urls:
print(internal_link.strip(), file=f)

# save the external links to a file
withopen(f"{domain_name}_external_links.txt", "w") asf:
forexternal_linkinexternal_urls:
print(external_link.strip(), file=f)

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
importrequests
fromurllib.parseimporturlparse, urljoin
frombs4importBeautifulSoup
importcolorama

# init the colorama module
colorama.init()

GREEN=colorama.Fore.GREEN
GRAY=colorama.Fore.LIGHTBLACK_EX
RESET=colorama.Fore.RESET

# initialize the set of links (unique links)
internal_urls=set()
external_urls=set()

total_urls_visited=0


defis_valid(url):
"""
 Checks whether `url` is a valid URL.
 """
parsed=urlparse(url)
returnbool(parsed.netloc) andbool(parsed.scheme)


defget_all_website_links(url):
"""
 Returns all URLs that is found on `url` in which it belongs to the same website
 """
# all URLs of `url`
urls=set()
# domain name of the URL without the protocol
domain_name=urlparse(url).netloc
soup=BeautifulSoup(requests.get(url).content, "html.parser")
fora_taginsoup.findAll("a"):
href=a_tag.attrs.get("href")
ifhref==""orhrefisNone:
# href empty tag
continue
# join the URL if it's relative (not absolute link)
href=urljoin(url, href)
parsed_href=urlparse(href)
# remove URL GET parameters, URL fragments, etc.
href=parsed_href.scheme+"://"+parsed_href.netloc+parsed_href.path
ifnotis_valid(href):
# not a valid URL
continue
ifhrefininternal_urls:
# already in the set
continue
ifdomain_namenotinhref:
# external link
ifhrefnotinexternal_urls:
print(f"{GRAY}[!] External link: {href}{RESET}")
external_urls.add(href)
continue
print(f"{GREEN}[*] Internal link: {href}{RESET}")
urls.add(href)
internal_urls.add(href)
returnurls


defcrawl(url, max_urls=50):
"""
 Crawls a web page and extracts all links.
 You'll find all links in `external_urls` and `internal_urls` global set variables.
 params:
 max_urls (int): number of max urls to crawl, default is 30.
 """
globaltotal_urls_visited
total_urls_visited+=1
links=get_all_website_links(url)
forlinkinlinks:
iftotal_urls_visited>max_urls:
break
crawl(link, max_urls=max_urls)


if__name__=="__main__":
importargparse
parser=argparse.ArgumentParser(description="Link Extractor Tool with Python")
parser.add_argument("url", help="The URL to extract links from.")
parser.add_argument("-m", "--max-urls", help="Number of max URLs to crawl, default is 30.", default=30, type=int)

args=parser.parse_args()
url=args.url
max_urls=args.max_urls

crawl(url, max_urls=max_urls)

print("[+] Total Internal links:", len(internal_urls))
print("[+] Total External links:", len(external_urls))
print("[+] Total URLs:", len(external_urls) +len(internal_urls))

domain_name=urlparse(url).netloc

# save the internal links to a file
withopen(f"{domain_name}_internal_links.txt", "w") asf:
forinternal_linkininternal_urls:
print(internal_link.strip(), file=f)

# save the external links to a file
withopen(f"{domain_name}_external_links.txt", "w") asf:
forexternal_linkinexternal_urls:
print(external_link.strip(), file=f)