SimpleSpider/SimpleHTML.py at master · ImprobabilityCast/SimpleSpider · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
fromcollectionsimportdeque
fromtypingimportIterable

importtokenizer
fromHTMLNodeimportHTMLNode


_zeroChildNames=frozenset({'meta', 'link', 'hr', 'br'})

# parses html text into a tree of html nodes
defparse(text: str) ->HTMLNode:
tokens=tokenizer.tokenize(text)
skipDoc(tokens)
returnparseNode(tokens)

# parses tokenized html text
# stopping at the first token following a '<' that isn't '!' or '?'
defskipDoc(chars: deque):
ifchars[0] =='<':
chars.popleft()
second=chars.popleft()
ifsecond=='?':
discardThruToStr(chars, '?>')
skipDoc(chars)
elifsecond=='!':
discardThruToStr(chars, '>')
skipDoc(chars)
else:
chars.appendleft(second)

defextractThruToStr(chars: deque, tokens) ->deque:
result=deque()
currentMatch=deque()
whilelen(chars) >0andlen(currentMatch) !=len(tokens):
ch=chars.popleft()
iftokens[len(currentMatch)] ==ch:
currentMatch.append(ch)
else:
currentMatch.clear()
result.append(ch)
returnresult

# removes all elements in 'chars' up to and including the first occurance of 'tokens'
defdiscardThruToStr(chars: deque, tokens):
currentMatch=0
whilelen(chars) >0andcurrentMatch!=len(tokens):
ch=chars.popleft()
iftokens[currentMatch] ==ch:
currentMatch+=1
else:
currentMatch=0

# '>' or '/>' must be a suffix of achars
defparseAttributes(achars: deque) ->dict:
result=dict()

whileachars[0] !='>'andachars[0] !='/':
name=achars.popleft()
value=deque()
ifachars[0] =='=':
achars.popleft()
sep=achars.popleft()
whileTrue:
partial=extractThruToStr(achars, sep)
partial.pop()
value.extend(partial)
iflen(partial) >0andvalue[-1] =='\\':
value.append(sep)
else:
break
result[name] =''.join(value)
returnresult

# length of chars must be > 0
defparseNode(chars: deque) ->HTMLNode:
node=HTMLNode(chars.popleft().lower())

# parse attributes
attributes=extractThruToStr(chars, '>')
node.attributes=parseAttributes(attributes)

# if not self-closing tag or something like a meta tag
ifattributes[0] =='>'andnode.namenotin_zeroChildNames:
# ignore script nodes
ifnode.name=='script'ornode.name=='style':
extractThruToStr(chars, ['<', '/', node.name, '>'])
returnnode

whilelen(chars) >0:
# skip text nodes
discardThruToStr(chars, '<')

ifchars[0] =='!':
# skip comments
discardThruToStr(chars, '-->')
elifchars[0] =='/':
discardThruToStr(chars, '>')
break
else:
node.childList.append(parseNode(chars))

returnnode