From 080b6ed33e6d94dbe250863b1335ca09c7abb554 Mon Sep 17 00:00:00 2001 From: Bret McGuire Date: Tue, 9 May 2023 09:29:35 -0500 Subject: [PATCH 1/7] Initial work --- cassandra/__init__.py | 2 +- cassandra/cqltypes.py | 30 ++++++++++++++++++++++++++++-- cassandra/protocol.py | 1 + tests/unit/test_types.py | 16 ++++++++++++++++ 4 files changed, 46 insertions(+), 3 deletions(-) diff --git a/cassandra/__init__.py b/cassandra/__init__.py index 1573abdf00..ca15e93602 100644 --- a/cassandra/__init__.py +++ b/cassandra/__init__.py @@ -22,7 +22,7 @@ def emit(self, record): logging.getLogger('cassandra').addHandler(NullHandler()) -__version_info__ = (3, 27, 0) +__version_info__ = (3, 28, 0b1) __version__ = '.'.join(map(str, __version_info__)) diff --git a/cassandra/cqltypes.py b/cassandra/cqltypes.py index 7946a63af8..898471d08d 100644 --- a/cassandra/cqltypes.py +++ b/cassandra/cqltypes.py @@ -213,6 +213,7 @@ def lookup_casstype_simple(casstype): def parse_casstype_args(typestring): + log.debug("parse_casstype_args: %s" % typestring) tokens, remainder = casstype_scanner.scan(typestring) if remainder: raise ValueError("weird characters %r at end" % remainder) @@ -235,7 +236,10 @@ def parse_casstype_args(typestring): else: names.append(None) - ctype = lookup_casstype_simple(tok) + try: + ctype = int(tok) + except ValueError: + ctype = lookup_casstype_simple(tok) types.append(ctype) # return the first (outer) type, which will have all parameters applied @@ -257,7 +261,9 @@ def lookup_casstype(casstype): if isinstance(casstype, (CassandraType, CassandraTypeType)): return casstype try: - return parse_casstype_args(casstype) + rv = parse_casstype_args(casstype) + #log.info("lookup_casstype rv: %s" % rv) + return rv except (ValueError, AssertionError, IndexError) as e: raise ValueError("Don't know how to parse type string %r: %s" % (casstype, e)) @@ -1421,3 +1427,23 @@ def serialize(cls, v, protocol_version): buf.write(int8_pack(cls._encode_precision(bound.precision))) return buf.getvalue() + +class VectorType(_CassandraType): + typename = 'org.apache.cassandra.db.marshal.VectorType' + + vector_size = 0 + + @classmethod + def apply_parameters(cls, subtypes, names): + cls.vector_size = subtypes + + @classmethod + def deserialize(cls, byts, protocol_version): + return [float_unpack(float_bytes) for float_bytes in (byts[i:i+4] for i in range(cls.vector_size))] + + @classmethod + def serialize(cls, v, protocol_version): + return None + + def __repr__(self): + return '<%s( %r )>' % (self.cql_parameterized_type(), self.vector_size) diff --git a/cassandra/protocol.py b/cassandra/protocol.py index 5e3610811e..d1b335f8dc 100644 --- a/cassandra/protocol.py +++ b/cassandra/protocol.py @@ -855,6 +855,7 @@ def recv_results_schema_change(self, f, protocol_version): @classmethod def read_type(cls, f, user_type_map): optid = read_short(f) + log.info("optid: %d" % optid) try: typeclass = cls.type_codes[optid] except KeyError: diff --git a/tests/unit/test_types.py b/tests/unit/test_types.py index af3b327ef8..90340369dd 100644 --- a/tests/unit/test_types.py +++ b/tests/unit/test_types.py @@ -190,6 +190,22 @@ class BarType(FooType): self.assertEqual(UTF8Type, ctype.subtypes[2]) self.assertEqual([b'city', None, b'zip'], ctype.names) + def test_parse_casstype_args_numeric(self): + class NumericParamType(CassandraType): + typename = 'org.apache.cassandra.db.marshal.NumericParamType' + + def __init__(self, subtypes, names): + self.subtypes = subtypes + self.names = names + + @classmethod + def apply_parameters(cls, subtypes, names): + return cls(subtypes, names) + + ctype = parse_casstype_args("org.apache.cassandra.db.marshal.NumericParamType(3)") + self.assertEqual(NumericParamType, ctype.__class__) + self.assertEqual(3, ctype.subtypes[0]) + def test_empty_value(self): self.assertEqual(str(EmptyValue()), 'EMPTY') From 4905dd8158db2cf174cec3ce9400a7b0becf0ccc Mon Sep 17 00:00:00 2001 From: Bret McGuire Date: Wed, 10 May 2023 15:38:22 -0500 Subject: [PATCH 2/7] Type machinery seems to be working well enough to support lookup_casstype/parse_castype_args for the vector type. If we can get serde working we might be in good shape. --- cassandra/cqltypes.py | 26 ++++++++++++++++---------- tests/unit/test_types.py | 21 +++++---------------- 2 files changed, 21 insertions(+), 26 deletions(-) diff --git a/cassandra/cqltypes.py b/cassandra/cqltypes.py index 898471d08d..05ccccf0bf 100644 --- a/cassandra/cqltypes.py +++ b/cassandra/cqltypes.py @@ -215,12 +215,14 @@ def lookup_casstype_simple(casstype): def parse_casstype_args(typestring): log.debug("parse_casstype_args: %s" % typestring) tokens, remainder = casstype_scanner.scan(typestring) + log.debug("tokens: %s, remainder: %s" % (tokens, remainder)) if remainder: raise ValueError("weird characters %r at end" % remainder) # use a stack of (types, names) lists args = [([], [])] for tok in tokens: + log.debug("Token: %s" % tok) if tok == '(': args.append(([], [])) elif tok == ')': @@ -240,10 +242,14 @@ def parse_casstype_args(typestring): ctype = int(tok) except ValueError: ctype = lookup_casstype_simple(tok) + log.debug("Appending %s to types" % ctype) types.append(ctype) # return the first (outer) type, which will have all parameters applied - return args[0][0][0] + log.info("args: %s" % args) + rv = args[0][0][0] + log.info("parse_casstype_args rv: %s" % rv) + return rv def lookup_casstype(casstype): @@ -262,9 +268,10 @@ def lookup_casstype(casstype): return casstype try: rv = parse_casstype_args(casstype) - #log.info("lookup_casstype rv: %s" % rv) + log.info("lookup_casstype rv: %s" % rv) return rv except (ValueError, AssertionError, IndexError) as e: + log.debug("Exception in parse_casstype_args: %s" % e) raise ValueError("Don't know how to parse type string %r: %s" % (casstype, e)) @@ -302,7 +309,7 @@ class _CassandraType(object): """ def __repr__(self): - return '<%s( %r )>' % (self.cql_parameterized_type(), self.val) + return '<%s>' % (self.cql_parameterized_type()) @classmethod def from_binary(cls, byts, protocol_version): @@ -1430,20 +1437,19 @@ def serialize(cls, v, protocol_version): class VectorType(_CassandraType): typename = 'org.apache.cassandra.db.marshal.VectorType' - vector_size = 0 @classmethod - def apply_parameters(cls, subtypes, names): - cls.vector_size = subtypes + def apply_parameters(cls, params, names): + log.debug("apply_paramters params: %s" % params) + assert len(params) == 1 + vsize = params[0] + return type('%s(%s)' % (cls.cass_parameterized_type_with([]), vsize), (cls,), {'vector_size': vsize}) @classmethod def deserialize(cls, byts, protocol_version): - return [float_unpack(float_bytes) for float_bytes in (byts[i:i+4] for i in range(cls.vector_size))] + return [float_unpack(bytes(float_bytes)) for float_bytes in (byts[i:i+4] for i in range(0, cls.vector_size, 4))] @classmethod def serialize(cls, v, protocol_version): return None - - def __repr__(self): - return '<%s( %r )>' % (self.cql_parameterized_type(), self.vector_size) diff --git a/tests/unit/test_types.py b/tests/unit/test_types.py index 90340369dd..842427066c 100644 --- a/tests/unit/test_types.py +++ b/tests/unit/test_types.py @@ -27,7 +27,7 @@ EmptyValue, LongType, SetType, UTF8Type, cql_typename, int8_pack, int64_pack, lookup_casstype, lookup_casstype_simple, parse_casstype_args, - int32_pack, Int32Type, ListType, MapType + int32_pack, Int32Type, ListType, MapType, VectorType ) from cassandra.encoder import cql_quote from cassandra.pool import Host @@ -190,21 +190,10 @@ class BarType(FooType): self.assertEqual(UTF8Type, ctype.subtypes[2]) self.assertEqual([b'city', None, b'zip'], ctype.names) - def test_parse_casstype_args_numeric(self): - class NumericParamType(CassandraType): - typename = 'org.apache.cassandra.db.marshal.NumericParamType' - - def __init__(self, subtypes, names): - self.subtypes = subtypes - self.names = names - - @classmethod - def apply_parameters(cls, subtypes, names): - return cls(subtypes, names) - - ctype = parse_casstype_args("org.apache.cassandra.db.marshal.NumericParamType(3)") - self.assertEqual(NumericParamType, ctype.__class__) - self.assertEqual(3, ctype.subtypes[0]) + def test_parse_casstype_vector(self): + ctype = parse_casstype_args("org.apache.cassandra.db.marshal.VectorType(3)") + self.assertTrue(issubclass(ctype, VectorType)) + self.assertEqual(3, ctype.vector_size) def test_empty_value(self): self.assertEqual(str(EmptyValue()), 'EMPTY') From f907b496c1c3de7e7d4c8dd6d25a5dcf55fbed2c Mon Sep 17 00:00:00 2001 From: Bret McGuire Date: Wed, 10 May 2023 16:43:33 -0500 Subject: [PATCH 3/7] Apparently got serialization sorted out --- cassandra/cqltypes.py | 8 ++++++-- tests/unit/test_types.py | 9 +++++++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/cassandra/cqltypes.py b/cassandra/cqltypes.py index 05ccccf0bf..9355bdc9bf 100644 --- a/cassandra/cqltypes.py +++ b/cassandra/cqltypes.py @@ -1448,8 +1448,12 @@ def apply_parameters(cls, params, names): @classmethod def deserialize(cls, byts, protocol_version): - return [float_unpack(bytes(float_bytes)) for float_bytes in (byts[i:i+4] for i in range(0, cls.vector_size, 4))] + indexes = (4 * x for x in range(0, cls.vector_size)) + return [float_unpack(byts[idx:idx + 4]) for idx in indexes] @classmethod def serialize(cls, v, protocol_version): - return None + buf = io.BytesIO() + for item in v: + buf.write(float_pack(item)) + return buf.getvalue() diff --git a/tests/unit/test_types.py b/tests/unit/test_types.py index 842427066c..86e0c63667 100644 --- a/tests/unit/test_types.py +++ b/tests/unit/test_types.py @@ -308,6 +308,15 @@ def test_cql_quote(self): self.assertEqual(cql_quote('test'), "'test'") self.assertEqual(cql_quote(0), '0') + def test_vector(self): + base = [3.4, 2.9, 41.6, 12.0] + ctype = parse_casstype_args("org.apache.cassandra.db.marshal.VectorType(4)") + base_bytes = ctype.serialize(base, 0) + self.assertEqual(16, len(base_bytes)) + result = ctype.deserialize(base_bytes, 0) + self.assertEqual(len(base), len(result)) + for idx in range(0,len(base)): + self.assertAlmostEqual(base[idx], result[idx], places=5) ZERO = datetime.timedelta(0) From c93edeb91ba2b5a09dca8a2b032377ff180e16b9 Mon Sep 17 00:00:00 2001 From: Bret McGuire Date: Wed, 10 May 2023 16:55:12 -0500 Subject: [PATCH 4/7] Removing lots of extraneous logging --- cassandra/cqltypes.py | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/cassandra/cqltypes.py b/cassandra/cqltypes.py index 9355bdc9bf..2f18f69ad8 100644 --- a/cassandra/cqltypes.py +++ b/cassandra/cqltypes.py @@ -213,9 +213,7 @@ def lookup_casstype_simple(casstype): def parse_casstype_args(typestring): - log.debug("parse_casstype_args: %s" % typestring) tokens, remainder = casstype_scanner.scan(typestring) - log.debug("tokens: %s, remainder: %s" % (tokens, remainder)) if remainder: raise ValueError("weird characters %r at end" % remainder) @@ -242,15 +240,10 @@ def parse_casstype_args(typestring): ctype = int(tok) except ValueError: ctype = lookup_casstype_simple(tok) - log.debug("Appending %s to types" % ctype) types.append(ctype) # return the first (outer) type, which will have all parameters applied - log.info("args: %s" % args) - rv = args[0][0][0] - log.info("parse_casstype_args rv: %s" % rv) - return rv - + return args[0][0][0] def lookup_casstype(casstype): """ @@ -267,9 +260,7 @@ def lookup_casstype(casstype): if isinstance(casstype, (CassandraType, CassandraTypeType)): return casstype try: - rv = parse_casstype_args(casstype) - log.info("lookup_casstype rv: %s" % rv) - return rv + return parse_casstype_args(casstype) except (ValueError, AssertionError, IndexError) as e: log.debug("Exception in parse_casstype_args: %s" % e) raise ValueError("Don't know how to parse type string %r: %s" % (casstype, e)) @@ -1441,7 +1432,6 @@ class VectorType(_CassandraType): @classmethod def apply_parameters(cls, params, names): - log.debug("apply_paramters params: %s" % params) assert len(params) == 1 vsize = params[0] return type('%s(%s)' % (cls.cass_parameterized_type_with([]), vsize), (cls,), {'vector_size': vsize}) From 33c91ca64e00ce3d905ece6d0490b56f5e412812 Mon Sep 17 00:00:00 2001 From: Bret McGuire Date: Thu, 11 May 2023 12:02:55 -0500 Subject: [PATCH 5/7] Removal of some additional logging statements --- cassandra/cqltypes.py | 1 - cassandra/protocol.py | 1 - 2 files changed, 2 deletions(-) diff --git a/cassandra/cqltypes.py b/cassandra/cqltypes.py index 2f18f69ad8..45d83ed8a6 100644 --- a/cassandra/cqltypes.py +++ b/cassandra/cqltypes.py @@ -220,7 +220,6 @@ def parse_casstype_args(typestring): # use a stack of (types, names) lists args = [([], [])] for tok in tokens: - log.debug("Token: %s" % tok) if tok == '(': args.append(([], [])) elif tok == ')': diff --git a/cassandra/protocol.py b/cassandra/protocol.py index d1b335f8dc..5e3610811e 100644 --- a/cassandra/protocol.py +++ b/cassandra/protocol.py @@ -855,7 +855,6 @@ def recv_results_schema_change(self, f, protocol_version): @classmethod def read_type(cls, f, user_type_map): optid = read_short(f) - log.info("optid: %d" % optid) try: typeclass = cls.type_codes[optid] except KeyError: From c83021d2004cd89eff11b084bc67220c2a817af4 Mon Sep 17 00:00:00 2001 From: Bret McGuire Date: Fri, 12 May 2023 14:48:20 -0500 Subject: [PATCH 6/7] Added cql_parameterized_type() method to help VectorType play better with cqlsh --- cassandra/cqltypes.py | 4 ++++ tests/unit/test_types.py | 6 +++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/cassandra/cqltypes.py b/cassandra/cqltypes.py index 45d83ed8a6..9cad7774d0 100644 --- a/cassandra/cqltypes.py +++ b/cassandra/cqltypes.py @@ -1446,3 +1446,7 @@ def serialize(cls, v, protocol_version): for item in v: buf.write(float_pack(item)) return buf.getvalue() + + @classmethod + def cql_parameterized_type(cls): + return "%s<%s>" % (cls.typename,cls.vector_size) diff --git a/tests/unit/test_types.py b/tests/unit/test_types.py index 86e0c63667..92c821bc40 100644 --- a/tests/unit/test_types.py +++ b/tests/unit/test_types.py @@ -308,7 +308,7 @@ def test_cql_quote(self): self.assertEqual(cql_quote('test'), "'test'") self.assertEqual(cql_quote(0), '0') - def test_vector(self): + def test_vector_round_trip(self): base = [3.4, 2.9, 41.6, 12.0] ctype = parse_casstype_args("org.apache.cassandra.db.marshal.VectorType(4)") base_bytes = ctype.serialize(base, 0) @@ -318,6 +318,10 @@ def test_vector(self): for idx in range(0,len(base)): self.assertAlmostEqual(base[idx], result[idx], places=5) + def test_vector_cql_parameterized_type(self): + ctype = parse_casstype_args("org.apache.cassandra.db.marshal.VectorType(4)") + self.assertEqual(ctype.cql_parameterized_type(), "org.apache.cassandra.db.marshal.VectorType<4>") + ZERO = datetime.timedelta(0) From e031256f790b6cd271023fec0b3942ae3d76f0a3 Mon Sep 17 00:00:00 2001 From: Bret McGuire Date: Thu, 25 May 2023 20:53:31 -0500 Subject: [PATCH 7/7] Update to work with new impl of CEP-7 type syntax --- cassandra/cqltypes.py | 14 ++++++++------ tests/unit/test_types.py | 12 +++++++----- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/cassandra/cqltypes.py b/cassandra/cqltypes.py index 9cad7774d0..6f98244c88 100644 --- a/cassandra/cqltypes.py +++ b/cassandra/cqltypes.py @@ -1428,25 +1428,27 @@ def serialize(cls, v, protocol_version): class VectorType(_CassandraType): typename = 'org.apache.cassandra.db.marshal.VectorType' vector_size = 0 + subtype = None @classmethod def apply_parameters(cls, params, names): - assert len(params) == 1 - vsize = params[0] - return type('%s(%s)' % (cls.cass_parameterized_type_with([]), vsize), (cls,), {'vector_size': vsize}) + assert len(params) == 2 + subtype = lookup_casstype(params[0]) + vsize = params[1] + return type('%s(%s)' % (cls.cass_parameterized_type_with([]), vsize), (cls,), {'vector_size': vsize, 'subtype': subtype}) @classmethod def deserialize(cls, byts, protocol_version): indexes = (4 * x for x in range(0, cls.vector_size)) - return [float_unpack(byts[idx:idx + 4]) for idx in indexes] + return [cls.subtype.deserialize(byts[idx:idx + 4], protocol_version) for idx in indexes] @classmethod def serialize(cls, v, protocol_version): buf = io.BytesIO() for item in v: - buf.write(float_pack(item)) + buf.write(cls.subtype.serialize(item, protocol_version)) return buf.getvalue() @classmethod def cql_parameterized_type(cls): - return "%s<%s>" % (cls.typename,cls.vector_size) + return "%s<%s, %s>" % (cls.typename, cls.subtype.typename, cls.vector_size) diff --git a/tests/unit/test_types.py b/tests/unit/test_types.py index 92c821bc40..e85f5dbe67 100644 --- a/tests/unit/test_types.py +++ b/tests/unit/test_types.py @@ -27,7 +27,8 @@ EmptyValue, LongType, SetType, UTF8Type, cql_typename, int8_pack, int64_pack, lookup_casstype, lookup_casstype_simple, parse_casstype_args, - int32_pack, Int32Type, ListType, MapType, VectorType + int32_pack, Int32Type, ListType, MapType, VectorType, + FloatType ) from cassandra.encoder import cql_quote from cassandra.pool import Host @@ -191,9 +192,10 @@ class BarType(FooType): self.assertEqual([b'city', None, b'zip'], ctype.names) def test_parse_casstype_vector(self): - ctype = parse_casstype_args("org.apache.cassandra.db.marshal.VectorType(3)") + ctype = parse_casstype_args("org.apache.cassandra.db.marshal.VectorType(org.apache.cassandra.db.marshal.FloatType, 3)") self.assertTrue(issubclass(ctype, VectorType)) self.assertEqual(3, ctype.vector_size) + self.assertEqual(FloatType, ctype.subtype) def test_empty_value(self): self.assertEqual(str(EmptyValue()), 'EMPTY') @@ -310,7 +312,7 @@ def test_cql_quote(self): def test_vector_round_trip(self): base = [3.4, 2.9, 41.6, 12.0] - ctype = parse_casstype_args("org.apache.cassandra.db.marshal.VectorType(4)") + ctype = parse_casstype_args("org.apache.cassandra.db.marshal.VectorType(org.apache.cassandra.db.marshal.FloatType, 4)") base_bytes = ctype.serialize(base, 0) self.assertEqual(16, len(base_bytes)) result = ctype.deserialize(base_bytes, 0) @@ -319,8 +321,8 @@ def test_vector_round_trip(self): self.assertAlmostEqual(base[idx], result[idx], places=5) def test_vector_cql_parameterized_type(self): - ctype = parse_casstype_args("org.apache.cassandra.db.marshal.VectorType(4)") - self.assertEqual(ctype.cql_parameterized_type(), "org.apache.cassandra.db.marshal.VectorType<4>") + ctype = parse_casstype_args("org.apache.cassandra.db.marshal.VectorType(org.apache.cassandra.db.marshal.FloatType, 4)") + self.assertEqual(ctype.cql_parameterized_type(), "org.apache.cassandra.db.marshal.VectorType") ZERO = datetime.timedelta(0)