From f5cfa077874eb186556ad03d56179561eab48bb8 Mon Sep 17 00:00:00 2001 From: Hideaki Takahashi Date: Wed, 1 Jul 2020 11:18:11 -0400 Subject: [PATCH] in FTS5, reject an APSW connection if amalgamation build is used --- README.rst | 5 ++++- sqlitefts/fts5.py | 6 ++++++ tests/test_apsw.py | 41 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 51 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 8d838e2..78f7327 100644 --- a/README.rst +++ b/README.rst @@ -72,7 +72,9 @@ Requirements * SQLite 3.10.2 and older versions do not have extra requirements. 2-arg fts3_tokenizer is always avaiable. * SQLite 3.12.0 and later vesrions do not have extra requirements. 2-arg fts3_tokenizer can be enabled dynamically. -Note for APSW users: An APSW Amalgamation build does not expose SQLite APIs used in this module, so libsqlite3.so/sqlite3.dll is also required even it has no runtime library dependencies on SQLite. An APSW local build already depends on the shared library. Detail: sqlite3_db_config can be invoked via Connection.config, but it rejects SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER to register a new tokenizer. tested at APSW 3.21.0-r1. +Note for APSW users: + * An APSW Amalgamation build does not expose SQLite APIs used in this module, so libsqlite3.so/sqlite3.dll is also required even it has no runtime library dependencies on SQLite. An APSW local build already depends on the shared library. Detail: sqlite3_db_config can be invoked via Connection.config, but it rejects SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER to register a new tokenizer. tested at APSW 3.21.0-r1. + * sqlitefts.fts5 does not support APSW Amalgamation build. see GH-14_ Licence ======= @@ -98,3 +100,4 @@ Thanks .. |appveyor build status| image:: https://ci.appveyor.com/api/projects/status/github/hideaki-t/sqlite-fts-python?svg=true .. _appveyor build status: https://ci.appveyor.com/project/hideaki-t/sqlite-fts-python .. _APSW: https://github.com/rogerbinns/apsw +.. _GH-14: https://github.com/hideaki-t/sqlite-fts-python/issues/14 diff --git a/sqlitefts/fts5.py b/sqlitefts/fts5.py index f0d445b..2ac638a 100644 --- a/sqlitefts/fts5.py +++ b/sqlitefts/fts5.py @@ -5,6 +5,7 @@ import struct from typing import Callable, Any, Union, Optional, TYPE_CHECKING, Iterable, Tuple, Dict +from .error import Error from .tokenizer import ffi, dll, get_db_from_connection, SQLITE_OK from .fts3 import Tokenizer as FTS3Tokenizer if TYPE_CHECKING: @@ -132,6 +133,11 @@ def tokenize(self, text: str, flags: int=0) -> Iterable[Tuple[str, int, int]]: def fts5_api_from_db(c: 'Union[sqlite3.Connection, apsw.Connection]'): + if not hasattr(c, 'commit'): + # APSW doesn't have conn.commit/rollback + import apsw + if apsw.using_amalgamation: + raise Error('unable to get fts5_api') cur = c.cursor() try: cur.execute('SELECT sqlite_version()') diff --git a/tests/test_apsw.py b/tests/test_apsw.py index faa4d30..98e742c 100644 --- a/tests/test_apsw.py +++ b/tests/test_apsw.py @@ -6,6 +6,7 @@ apsw = pytest.importorskip('apsw') import sqlitefts as fts +from sqlitefts import fts5, fts5_aux class SimpleTokenizer(fts.Tokenizer): @@ -20,6 +21,18 @@ def tokenize(self, text): yield t, p, p + l +class SimpleFTS5Tokenizer(fts5.FTS5Tokenizer): + _p = re.compile(r'\w+', re.UNICODE) + + def tokenize(self, text, flags): + for m in self._p.finditer(text): + s, e = m.span() + t = text[s:e] + l = len(t.encode('utf-8')) + p = len(text[:s].encode('utf-8')) + yield t, p, p + l + + def test_createtable(): c = apsw.Connection(':memory:') name = 'simple' @@ -258,3 +271,31 @@ def test_tokenizer_output(): "SELECT token, start, end, position " "FROM tok1 WHERE input=?", [s]), expect): assert e == a + + +@pytest.mark.xfail(apsw.using_amalgamation, + reason='FTS5 with APSW+Amalgamation not supported') +def test_fts5_api_from_db(): + with apsw.Connection(':memory:') as c: + fts5api = fts5.fts5_api_from_db(c) + assert fts5api.iVersion == 2 + assert fts5api.xCreateTokenizer + + +@pytest.mark.xfail(apsw.using_amalgamation, + reason='FTS5 with APSW+Amalgamation not supported', + raises=fts.Error) +def test_aux_and_tokenize(): + c = apsw.Connection(':memory:') + try: + fts5_aux.register_aux_function(c, 'tokenize', fts5_aux.aux_tokenize) + cur = c.cursor() + cur.execute('CREATE VIRTUAL TABLE fts USING FTS5(content)') + cur.executemany('INSERT INTO fts VALUES(?)', + (['hello world'], ['こんにちは 世界'])) + cur.execute('SELECT COUNT(*) FROM fts') + assert 2 == cur.fetchone()[0] + cur.execute('SELECT tokenize(fts, 0) FROM fts') + assert [x[0] for x in cur.fetchall()] == ['hello, world', 'こんにちは, 世界'] + finally: + c.close()