pythoncode-tutorials/web-scraping/html-table-extractor/html_table_extractor.py at master · hulinhui-code/pythoncode-tutorials · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
importrequests
importpandasaspd
frombs4importBeautifulSoupasbs

USER_AGENT="Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.157 Safari/537.36"
# US english
LANGUAGE="en-US,en;q=0.5"

defget_soup(url):
"""Constructs and returns a soup using the HTML content of `url` passed"""
# initialize a session
session=requests.Session()
# set the User-Agent as a regular browser
session.headers['User-Agent'] =USER_AGENT
# request for english content (optional)
session.headers['Accept-Language'] =LANGUAGE
session.headers['Content-Language'] =LANGUAGE
# make the request
html=session.get(url)
# return the soup
returnbs(html.content, "html.parser")


defget_all_tables(soup):
"""Extracts and returns all tables in a soup object"""
returnsoup.find_all("table")


defget_table_headers(table):
"""Given a table soup, returns all the headers"""
headers= []
forthintable.find("tr").find_all("th"):
headers.append(th.text.strip())
returnheaders


defget_table_rows(table):
"""Given a table, returns all its rows"""
rows= []
fortrintable.find_all("tr")[1:]:
cells= []
# grab all td tags in this table row
tds=tr.find_all("td")
iflen(tds) ==0:
# if no td tags, search for th tags
# can be found especially in wikipedia tables below the table
ths=tr.find_all("th")
forthinths:
cells.append(th.text.strip())
else:
# use regular td tags
fortdintds:
cells.append(td.text.strip())
rows.append(cells)
returnrows


defsave_as_csv(table_name, headers, rows):
pd.DataFrame(rows, columns=headers).to_csv(f"{table_name}.csv")


defmain(url):
# get the soup
soup=get_soup(url)
# extract all the tables from the web page
tables=get_all_tables(soup)
print(f"[+] Found a total of {len(tables)} tables.")
# iterate over all tables
fori, tableinenumerate(tables, start=1):
# get the table headers
headers=get_table_headers(table)
# get all the rows of the table
rows=get_table_rows(table)
# save table as csv file
table_name=f"table-{i}"
print(f"[+] Saving {table_name}")
save_as_csv(table_name, headers, rows)


if__name__=="__main__":
importsys
try:
url=sys.argv[1]
exceptIndexError:
print("Please specify a URL.\nUsage: python html_table_extractor.py [URL]")
exit(1)
main(url)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
importrequests
importpandasaspd
frombs4importBeautifulSoupasbs

USER_AGENT="Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.157 Safari/537.36"
# US english
LANGUAGE="en-US,en;q=0.5"

defget_soup(url):
"""Constructs and returns a soup using the HTML content of `url` passed"""
# initialize a session
session=requests.Session()
# set the User-Agent as a regular browser
session.headers['User-Agent'] =USER_AGENT
# request for english content (optional)
session.headers['Accept-Language'] =LANGUAGE
session.headers['Content-Language'] =LANGUAGE
# make the request
html=session.get(url)
# return the soup
returnbs(html.content, "html.parser")


defget_all_tables(soup):
"""Extracts and returns all tables in a soup object"""
returnsoup.find_all("table")


defget_table_headers(table):
"""Given a table soup, returns all the headers"""
headers= []
forthintable.find("tr").find_all("th"):
headers.append(th.text.strip())
returnheaders


defget_table_rows(table):
"""Given a table, returns all its rows"""
rows= []
fortrintable.find_all("tr")[1:]:
cells= []
# grab all td tags in this table row
tds=tr.find_all("td")
iflen(tds) ==0:
# if no td tags, search for th tags
# can be found especially in wikipedia tables below the table
ths=tr.find_all("th")
forthinths:
cells.append(th.text.strip())
else:
# use regular td tags
fortdintds:
cells.append(td.text.strip())
rows.append(cells)
returnrows


defsave_as_csv(table_name, headers, rows):
pd.DataFrame(rows, columns=headers).to_csv(f"{table_name}.csv")


defmain(url):
# get the soup
soup=get_soup(url)
# extract all the tables from the web page
tables=get_all_tables(soup)
print(f"[+] Found a total of {len(tables)} tables.")
# iterate over all tables
fori, tableinenumerate(tables, start=1):
# get the table headers
headers=get_table_headers(table)
# get all the rows of the table
rows=get_table_rows(table)
# save table as csv file
table_name=f"table-{i}"
print(f"[+] Saving {table_name}")
save_as_csv(table_name, headers, rows)


if__name__=="__main__":
importsys
try:
url=sys.argv[1]
exceptIndexError:
print("Please specify a URL.\nUsage: python html_table_extractor.py [URL]")
exit(1)
main(url)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
importrequests
importpandasaspd
frombs4importBeautifulSoupasbs

USER_AGENT="Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.157 Safari/537.36"
# US english
LANGUAGE="en-US,en;q=0.5"

defget_soup(url):
"""Constructs and returns a soup using the HTML content of `url` passed"""
# initialize a session
session=requests.Session()
# set the User-Agent as a regular browser
session.headers['User-Agent'] =USER_AGENT
# request for english content (optional)
session.headers['Accept-Language'] =LANGUAGE
session.headers['Content-Language'] =LANGUAGE
# make the request
html=session.get(url)
# return the soup
returnbs(html.content, "html.parser")


defget_all_tables(soup):
"""Extracts and returns all tables in a soup object"""
returnsoup.find_all("table")


defget_table_headers(table):
"""Given a table soup, returns all the headers"""
headers= []
forthintable.find("tr").find_all("th"):
headers.append(th.text.strip())
returnheaders


defget_table_rows(table):
"""Given a table, returns all its rows"""
rows= []
fortrintable.find_all("tr")[1:]:
cells= []
# grab all td tags in this table row
tds=tr.find_all("td")
iflen(tds) ==0:
# if no td tags, search for th tags
# can be found especially in wikipedia tables below the table
ths=tr.find_all("th")
forthinths:
cells.append(th.text.strip())
else:
# use regular td tags
fortdintds:
cells.append(td.text.strip())
rows.append(cells)
returnrows


defsave_as_csv(table_name, headers, rows):
pd.DataFrame(rows, columns=headers).to_csv(f"{table_name}.csv")


defmain(url):
# get the soup
soup=get_soup(url)
# extract all the tables from the web page
tables=get_all_tables(soup)
print(f"[+] Found a total of {len(tables)} tables.")
# iterate over all tables
fori, tableinenumerate(tables, start=1):
# get the table headers
headers=get_table_headers(table)
# get all the rows of the table
rows=get_table_rows(table)
# save table as csv file
table_name=f"table-{i}"
print(f"[+] Saving {table_name}")
save_as_csv(table_name, headers, rows)


if__name__=="__main__":
importsys
try:
url=sys.argv[1]
exceptIndexError:
print("Please specify a URL.\nUsage: python html_table_extractor.py [URL]")
exit(1)
main(url)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
importrequests
importpandasaspd
frombs4importBeautifulSoupasbs

USER_AGENT="Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.157 Safari/537.36"
# US english
LANGUAGE="en-US,en;q=0.5"

defget_soup(url):
"""Constructs and returns a soup using the HTML content of `url` passed"""
# initialize a session
session=requests.Session()
# set the User-Agent as a regular browser
session.headers['User-Agent'] =USER_AGENT
# request for english content (optional)
session.headers['Accept-Language'] =LANGUAGE
session.headers['Content-Language'] =LANGUAGE
# make the request
html=session.get(url)
# return the soup
returnbs(html.content, "html.parser")


defget_all_tables(soup):
"""Extracts and returns all tables in a soup object"""
returnsoup.find_all("table")


defget_table_headers(table):
"""Given a table soup, returns all the headers"""
headers= []
forthintable.find("tr").find_all("th"):
headers.append(th.text.strip())
returnheaders


defget_table_rows(table):
"""Given a table, returns all its rows"""
rows= []
fortrintable.find_all("tr")[1:]:
cells= []
# grab all td tags in this table row
tds=tr.find_all("td")
iflen(tds) ==0:
# if no td tags, search for th tags
# can be found especially in wikipedia tables below the table
ths=tr.find_all("th")
forthinths:
cells.append(th.text.strip())
else:
# use regular td tags
fortdintds:
cells.append(td.text.strip())
rows.append(cells)
returnrows


defsave_as_csv(table_name, headers, rows):
pd.DataFrame(rows, columns=headers).to_csv(f"{table_name}.csv")


defmain(url):
# get the soup
soup=get_soup(url)
# extract all the tables from the web page
tables=get_all_tables(soup)
print(f"[+] Found a total of {len(tables)} tables.")
# iterate over all tables
fori, tableinenumerate(tables, start=1):
# get the table headers
headers=get_table_headers(table)
# get all the rows of the table
rows=get_table_rows(table)
# save table as csv file
table_name=f"table-{i}"
print(f"[+] Saving {table_name}")
save_as_csv(table_name, headers, rows)


if__name__=="__main__":
importsys
try:
url=sys.argv[1]
exceptIndexError:
print("Please specify a URL.\nUsage: python html_table_extractor.py [URL]")
exit(1)
main(url)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
importrequests
importpandasaspd
frombs4importBeautifulSoupasbs

USER_AGENT="Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.157 Safari/537.36"
# US english
LANGUAGE="en-US,en;q=0.5"

defget_soup(url):
"""Constructs and returns a soup using the HTML content of `url` passed"""
# initialize a session
session=requests.Session()
# set the User-Agent as a regular browser
session.headers['User-Agent'] =USER_AGENT
# request for english content (optional)
session.headers['Accept-Language'] =LANGUAGE
session.headers['Content-Language'] =LANGUAGE
# make the request
html=session.get(url)
# return the soup
returnbs(html.content, "html.parser")


defget_all_tables(soup):
"""Extracts and returns all tables in a soup object"""
returnsoup.find_all("table")


defget_table_headers(table):
"""Given a table soup, returns all the headers"""
headers= []
forthintable.find("tr").find_all("th"):
headers.append(th.text.strip())
returnheaders


defget_table_rows(table):
"""Given a table, returns all its rows"""
rows= []
fortrintable.find_all("tr")[1:]:
cells= []
# grab all td tags in this table row
tds=tr.find_all("td")
iflen(tds) ==0:
# if no td tags, search for th tags
# can be found especially in wikipedia tables below the table
ths=tr.find_all("th")
forthinths:
cells.append(th.text.strip())
else:
# use regular td tags
fortdintds:
cells.append(td.text.strip())
rows.append(cells)
returnrows


defsave_as_csv(table_name, headers, rows):
pd.DataFrame(rows, columns=headers).to_csv(f"{table_name}.csv")


defmain(url):
# get the soup
soup=get_soup(url)
# extract all the tables from the web page
tables=get_all_tables(soup)
print(f"[+] Found a total of {len(tables)} tables.")
# iterate over all tables
fori, tableinenumerate(tables, start=1):
# get the table headers
headers=get_table_headers(table)
# get all the rows of the table
rows=get_table_rows(table)
# save table as csv file
table_name=f"table-{i}"
print(f"[+] Saving {table_name}")
save_as_csv(table_name, headers, rows)


if__name__=="__main__":
importsys
try:
url=sys.argv[1]
exceptIndexError:
print("Please specify a URL.\nUsage: python html_table_extractor.py [URL]")
exit(1)
main(url)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
importrequests
importpandasaspd
frombs4importBeautifulSoupasbs

USER_AGENT="Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.157 Safari/537.36"
# US english
LANGUAGE="en-US,en;q=0.5"

defget_soup(url):
"""Constructs and returns a soup using the HTML content of `url` passed"""
# initialize a session
session=requests.Session()
# set the User-Agent as a regular browser
session.headers['User-Agent'] =USER_AGENT
# request for english content (optional)
session.headers['Accept-Language'] =LANGUAGE
session.headers['Content-Language'] =LANGUAGE
# make the request
html=session.get(url)
# return the soup
returnbs(html.content, "html.parser")


defget_all_tables(soup):
"""Extracts and returns all tables in a soup object"""
returnsoup.find_all("table")


defget_table_headers(table):
"""Given a table soup, returns all the headers"""
headers= []
forthintable.find("tr").find_all("th"):
headers.append(th.text.strip())
returnheaders


defget_table_rows(table):
"""Given a table, returns all its rows"""
rows= []
fortrintable.find_all("tr")[1:]:
cells= []
# grab all td tags in this table row
tds=tr.find_all("td")
iflen(tds) ==0:
# if no td tags, search for th tags
# can be found especially in wikipedia tables below the table
ths=tr.find_all("th")
forthinths:
cells.append(th.text.strip())
else:
# use regular td tags
fortdintds:
cells.append(td.text.strip())
rows.append(cells)
returnrows


defsave_as_csv(table_name, headers, rows):
pd.DataFrame(rows, columns=headers).to_csv(f"{table_name}.csv")


defmain(url):
# get the soup
soup=get_soup(url)
# extract all the tables from the web page
tables=get_all_tables(soup)
print(f"[+] Found a total of {len(tables)} tables.")
# iterate over all tables
fori, tableinenumerate(tables, start=1):
# get the table headers
headers=get_table_headers(table)
# get all the rows of the table
rows=get_table_rows(table)
# save table as csv file
table_name=f"table-{i}"
print(f"[+] Saving {table_name}")
save_as_csv(table_name, headers, rows)


if__name__=="__main__":
importsys
try:
url=sys.argv[1]
exceptIndexError:
print("Please specify a URL.\nUsage: python html_table_extractor.py [URL]")
exit(1)
main(url)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
importrequests
importpandasaspd
frombs4importBeautifulSoupasbs

USER_AGENT="Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.157 Safari/537.36"
# US english
LANGUAGE="en-US,en;q=0.5"

defget_soup(url):
"""Constructs and returns a soup using the HTML content of `url` passed"""
# initialize a session
session=requests.Session()
# set the User-Agent as a regular browser
session.headers['User-Agent'] =USER_AGENT
# request for english content (optional)
session.headers['Accept-Language'] =LANGUAGE
session.headers['Content-Language'] =LANGUAGE
# make the request
html=session.get(url)
# return the soup
returnbs(html.content, "html.parser")


defget_all_tables(soup):
"""Extracts and returns all tables in a soup object"""
returnsoup.find_all("table")


defget_table_headers(table):
"""Given a table soup, returns all the headers"""
headers= []
forthintable.find("tr").find_all("th"):
headers.append(th.text.strip())
returnheaders


defget_table_rows(table):
"""Given a table, returns all its rows"""
rows= []
fortrintable.find_all("tr")[1:]:
cells= []
# grab all td tags in this table row
tds=tr.find_all("td")
iflen(tds) ==0:
# if no td tags, search for th tags
# can be found especially in wikipedia tables below the table
ths=tr.find_all("th")
forthinths:
cells.append(th.text.strip())
else:
# use regular td tags
fortdintds:
cells.append(td.text.strip())
rows.append(cells)
returnrows


defsave_as_csv(table_name, headers, rows):
pd.DataFrame(rows, columns=headers).to_csv(f"{table_name}.csv")


defmain(url):
# get the soup
soup=get_soup(url)
# extract all the tables from the web page
tables=get_all_tables(soup)
print(f"[+] Found a total of {len(tables)} tables.")
# iterate over all tables
fori, tableinenumerate(tables, start=1):
# get the table headers
headers=get_table_headers(table)
# get all the rows of the table
rows=get_table_rows(table)
# save table as csv file
table_name=f"table-{i}"
print(f"[+] Saving {table_name}")
save_as_csv(table_name, headers, rows)


if__name__=="__main__":
importsys
try:
url=sys.argv[1]
exceptIndexError:
print("Please specify a URL.\nUsage: python html_table_extractor.py [URL]")
exit(1)
main(url)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
importrequests
importpandasaspd
frombs4importBeautifulSoupasbs

USER_AGENT="Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.157 Safari/537.36"
# US english
LANGUAGE="en-US,en;q=0.5"

defget_soup(url):
"""Constructs and returns a soup using the HTML content of `url` passed"""
# initialize a session
session=requests.Session()
# set the User-Agent as a regular browser
session.headers['User-Agent'] =USER_AGENT
# request for english content (optional)
session.headers['Accept-Language'] =LANGUAGE
session.headers['Content-Language'] =LANGUAGE
# make the request
html=session.get(url)
# return the soup
returnbs(html.content, "html.parser")


defget_all_tables(soup):
"""Extracts and returns all tables in a soup object"""
returnsoup.find_all("table")


defget_table_headers(table):
"""Given a table soup, returns all the headers"""
headers= []
forthintable.find("tr").find_all("th"):
headers.append(th.text.strip())
returnheaders


defget_table_rows(table):
"""Given a table, returns all its rows"""
rows= []
fortrintable.find_all("tr")[1:]:
cells= []
# grab all td tags in this table row
tds=tr.find_all("td")
iflen(tds) ==0:
# if no td tags, search for th tags
# can be found especially in wikipedia tables below the table
ths=tr.find_all("th")
forthinths:
cells.append(th.text.strip())
else:
# use regular td tags
fortdintds:
cells.append(td.text.strip())
rows.append(cells)
returnrows


defsave_as_csv(table_name, headers, rows):
pd.DataFrame(rows, columns=headers).to_csv(f"{table_name}.csv")


defmain(url):
# get the soup
soup=get_soup(url)
# extract all the tables from the web page
tables=get_all_tables(soup)
print(f"[+] Found a total of {len(tables)} tables.")
# iterate over all tables
fori, tableinenumerate(tables, start=1):
# get the table headers
headers=get_table_headers(table)
# get all the rows of the table
rows=get_table_rows(table)
# save table as csv file
table_name=f"table-{i}"
print(f"[+] Saving {table_name}")
save_as_csv(table_name, headers, rows)


if__name__=="__main__":
importsys
try:
url=sys.argv[1]
exceptIndexError:
print("Please specify a URL.\nUsage: python html_table_extractor.py [URL]")
exit(1)
main(url)