Compile html5lib with Cython

.gitmodules

@@ -1,3 +1,3 @@
 [submodule "testdata"]
-	path = html5lib/tests/testdata
+	path = tests/testdata
 	url = https://github.com/html5lib/html5lib-tests.git

MANIFEST.in

@@ -7,4 +7,5 @@ include .pytest.expect
 include tox.ini
 include pytest.ini
 graft html5lib/tests/testdata
+recursive-include html5lib *.pxd
 recursive-include html5lib/tests *.py

benchmarks/bench_html.py

@@ -4,7 +4,7 @@
 
 import pyperf
 
-sys.path[0:0] = [os.path.join(os.path.dirname(__file__), "..")]
+#sys.path[0:0] = [os.path.join(os.path.dirname(__file__), "..")]
 import html5lib # noqa: E402
 
 
@@ -49,9 +49,9 @@ def add_cmdline_args(cmd, args):
  source = io.BytesIO(fh.read())
 
  if "parse" in benchmarks:
- for tb in ("etree", "dom", "lxml"):
+ for tb in ("etree",):
  runner.bench_func("html_parse_%s" % tb, bench_parse, source, tb)
 
- if "serialize" in benchmarks:
- for tb in ("etree", "dom", "lxml"):
- runner.bench_time_func("html_serialize_%s" % tb, bench_serialize, source, tb)
+# if "serialize" in benchmarks:
+#for tb in ("etree",):
+#  runner.bench_time_func("html_serialize_%s" % tb, bench_serialize, source, tb)

benchmarks/bench_wpt.py

@@ -4,7 +4,7 @@
 
 import pyperf
 
-sys.path[0:0] = [os.path.join(os.path.dirname(__file__), "..")]
+#sys.path[0:0] = [os.path.join(os.path.dirname(__file__), "..")]
 import html5lib # noqa: E402
 
 

html5lib/_ascii.py

@@ -0,0 +1,5 @@
+from .constants import asciiUpper2Lower
+
+
+def ascii_lower(s):
+ return s.translate(asciiUpper2Lower)

html5lib/_inputstream.pxd

@@ -0,0 +1,109 @@
+# cython: language_level=3
+cimport cython
+from cpython cimport array
+
+ctypedef void (*rCEf)(HTMLUnicodeInputStream, unicode) except *
+
+cdef dict charsUntilCache
+
+cdef class BufferedStream(object):
+ cdef object stream
+ cdef object buffer
+ cdef object position
+ cpdef object tell(self)
+ cpdef object seek(self, object pos)
+ cpdef object read(self, object bytes)
+ cdef object _bufferedBytes(self)
+ cdef object _readStream(self, object bytes)
+ cdef object _readFromBuffer(self, object bytes)
+
+#def HTMLInputStream(source, object **kwargs)
+
+cdef class HTMLUnicodeInputStream(object):
+ cdef rCEf reportCharacterErrors
+ cdef object newLines
+ cdef readonly object charEncoding
+ cdef object dataStream
+ cdef unicode chunk
+ cdef Py_ssize_t chunkSize
+ cdef Py_ssize_t chunkOffset
+ cdef readonly list errors
+
+ # number of (complete) lines in previous chunks
+ cdef Py_ssize_t prevNumLines
+ # number of columns in the last line of the previous chunk
+ cdef Py_ssize_t prevNumCols
+
+ # Deal with CR LF and surrogates split over chunk boundaries
+ cdef unicode _bufferedCharacter
+
+ cdef object reset(self)
+ cdef object openStream(self, object source)
+
+ @cython.locals(nLines=Py_ssize_t, lastLinePos=Py_ssize_t)
+ cdef tuple _position(self, Py_ssize_t offset)
+ cpdef tuple position(self)
+
+ @cython.locals(chunkOffset=Py_ssize_t, char=unicode)
+ cpdef unicode char(self)
+
+ @cython.locals(data=unicode)
+ cdef bint readChunk(self, Py_ssize_t chunkSize=?) except? -1
+
+ @cython.locals(c=ulong)
+ cdef void characterErrorsUCS4(self, unicode data) except *
+ cdef void characterErrorsUCS2(self, unicode data) except *
+
+ cpdef object charsUntil(self, object characters, bint opposite=?)
+ cpdef object unget(self, object char)
+
+cdef class HTMLBinaryInputStream(HTMLUnicodeInputStream):
+ cdef object rawStream
+ cdef readonly object numBytesMeta
+ cdef readonly object numBytesChardet
+ cdef object override_encoding
+ cdef object transport_encoding
+ cdef object same_origin_parent_encoding
+ cdef object likely_encoding
+ cdef object default_encoding
+ cdef object reset(self)
+ cdef object openStream(self, object source)
+ cdef object determineEncoding(self, object chardet=?)
+ cpdef object changeEncoding(self, object newEncoding)
+ @cython.locals(string=bytes)
+ cdef object detectBOM(self)
+ cdef object detectEncodingMeta(self)
+
+# cdef class EncodingBytes(bytes):
+# cdef object previous(self)
+# cdef object setPosition(self, object position)
+# cdef object getPosition(self)
+# cdef object getCurrentByte(self)
+# cdef object skip(self, object chars=?)
+# cdef object skipUntil(self, object chars)
+# cdef object matchBytes(self, object bytes)
+# cdef object jumpTo(self, object bytes)
+
+ctypedef bint (*encstate)(EncodingParser) except? -1
+
+cdef class EncodingParser(object):
+ cdef object data
+ cdef object encoding
+
+ @cython.locals(func=encstate, keepParsing=bint)
+ cdef object getEncoding(self)
+ cdef bint handleComment(self) except? -1
+ @cython.locals(hasPragma=bint, name=bytes, value=bytes, tentativeEncoding=bytes)
+ cdef bint handleMeta(self) except? -1
+ cdef bint handlePossibleStartTag(self) except? -1
+ cdef bint handlePossibleEndTag(self) except? -1
+ cdef bint handlePossibleTag(self, bint endTag) except? -1
+ cdef bint handleOther(self) except? -1
+ @cython.locals(c=bytes)
+ cdef tuple getAttribute(self)
+
+cdef class ContentAttrParser(object):
+ cdef object data
+ cpdef object parse(self) # this needs to be cpdef for tests
+
+cdef object lookupEncoding(object encoding)