SimpleSpider/SimpleHTMLTest.py at master · ImprobabilityCast/SimpleSpider · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52

# 3rd party lib
importrequests

fromsimpleHTMLimportparse

defprintTabs(num):
s=""
whilenum>0:
s+=' '
num-=1
returns

defprintNodes(node, indent):

print(printTabs(indent) +'<'+node.name+' '+str(node.attributes) +'>')
forninnode.childList:
printNodes(n, indent+1)
print(printTabs(indent) +'</'+node.name+'>')

defprintLinks(node):
nodes=node.getElementsByTagNames("a")
toPrint= []
forninnodes:
# if 'href' in n.attributes and n.attributes['href'] not in toPrint:
toPrint.append(n.attributes['href'])
# print(n.attributes['href'])
print(len(toPrint))


defmain():
# tests:
# http://localhost/php-manual-en/function.filter-input.html
# http://localhost/
# "<html><body><div><div></div></div></body></html>"
# "<html><body><div content pigs='null' turkey='' knife='egg\\'bacon'><p></p></div></body></html>"
#
# these make it parse slow:
# http://localhost/php-manual-en/indexes.examples.html
# http://localhost/php-manual-en/indexes.functions.html
#

response2=requests.get("http://localhost/php-manual-en/indexes.examples.html")

print("Parsing...")

doc=parse(response2.text)
printNodes(doc, 0)
printLinks(doc)

if__name__=="__main__":
main()