From 45705f6b4491fb9f7db1793a6f8bd4ac1a9edb5f Mon Sep 17 00:00:00 2001 From: Bret McGuire Date: Tue, 25 Apr 2023 17:12:44 -0500 Subject: [PATCH 1/5] Complete docstrings for everything in the policy --- cassandra/policies.py | 42 +++++++++++++++++++++++++++++++++++++----- 1 file changed, 37 insertions(+), 5 deletions(-) diff --git a/cassandra/policies.py b/cassandra/policies.py index 9d6c087362..3a0160543e 100644 --- a/cassandra/policies.py +++ b/cassandra/policies.py @@ -1195,19 +1195,54 @@ def _rethrow(self, *args, **kwargs): ColData = namedtuple('ColData', ['key','type']) class ColumnEncryptionPolicy(object): + """ + A policy enabling (mostly) transparent encryption and decryption of data before it is + sent to the cluster. + + Key materials and other configurations are specified on a per-column basis. This policy can + then be used by driver structures which are aware of the underlying columns involved in their + work. In practice this includes the following cases: + + * Prepared statements - data for columns specified by the cluster's policy will be transparently + encrypted before they are sent + * Rows returned from any query - data for columns specified by the cluster's policy will be + transparently decrypted before they are returned to the user + + To enable this functionality, create an instance of this class (or more likely a subclass) + before creating a cluster. This policy should then be configured and supplied to the Cluster + at creation time via the :attr:`.Cluster.column_encryption_policy` attribute. + """ + def encrypt(self, coldesc, obj_bytes): + """ + Encrypt the specified bytes using the cryptography materials for the specified column. + Largely used internally, although this could also be used to encrypt values supplied + to non-prepared statements in a way that is consistent with this policy. + """ raise NotImplementedError() def decrypt(self, coldesc, encrypted_bytes): + """ + Decrypt the specified (encrypted) bytes using the cryptography materials for the + specified column. Used internally; could be used externally as well but there's + not currently an obvious use case. + """ raise NotImplementedError() def add_column(self, coldesc, key): + """ + Provide cryptography materials to be used when encrypted and/or decrypting data + for the specified column. + """ raise NotImplementedError() def contains_column(self, coldesc): + """ + Predicate to determine if a specific column is supported by this policy. + Currently only used interally. + """ raise NotImplementedError() -# Both sizes below in AES256_BLOCK_SIZE = 128 AES256_BLOCK_SIZE_BYTES = int(AES256_BLOCK_SIZE / 8) AES256_KEY_SIZE = 256 @@ -1288,10 +1323,7 @@ def _get_cipher(self, coldesc): except KeyError: raise ValueError("Could not find column {}".format(coldesc)) + # Explicitly use a class method here to avoid caching self @lru_cache(maxsize=128) def _build_cipher(key, mode, iv): - """ - Explicitly use a class method here to avoid caching self - """ - return Cipher(algorithms.AES256(key), mode(iv)) From 281d4cbda7abba7b7b6a30afed2d633c8e485631 Mon Sep 17 00:00:00 2001 From: Bret McGuire Date: Wed, 26 Apr 2023 22:55:12 -0500 Subject: [PATCH 2/5] Initial take on docs --- docs/column_encryption.rst | 77 ++++++++++++++++++++++++++++++++++++++ docs/index.rst | 3 ++ 2 files changed, 80 insertions(+) create mode 100644 docs/column_encryption.rst diff --git a/docs/column_encryption.rst b/docs/column_encryption.rst new file mode 100644 index 0000000000..ca591a0e3e --- /dev/null +++ b/docs/column_encryption.rst @@ -0,0 +1,77 @@ +Column Encryption +================= + +Overview +-------- +Support for client-side encryption of data was added in version 3.27.0 of the Python driver. When using +this feature data will be encrypted on-the-fly according to a specified :class:`~.ColumnEncryptionPolicy` +instance. This policy is also used to decrypt data in returned rows. If a prepared statement is used +this decryption is transparent to the user; retrieved data will be decrypted and converted into the original +type (according to definitions in the encryption policy). Support for simple (i.e. non-prepared) queries is +also available, although in this case values must be manually encrypted and/or decrypted. The +:class:`~.ColumnEncryptionPolicy` instance provides methods to assist with these operations. + +Client-side encryption and decryption should work against all versions of Cassandra and DSE. It does not +utilize any server-side functionality to do it's work. + +Configuration +------------- +Client-side encryption is enabled by creating an instance of a subclass of :class:`~.ColumnEncryptionPolicy` +and adding information about columns to be encrypted to it. This policy is then supplied to :class:`~.Cluster` +when it's created. + +.. code-block:: python + import os + + from cassandra.policies import ColDesc, AES256ColumnEncryptionPolicy, AES256_KEY_SIZE_BYTES + + key = os.urandom(AES256_KEY_SIZE_BYTES) + cl_policy = AES256ColumnEncryptionPolicy() + col_desc = ColDesc('ks1','table1','column1') + cql_type = "int" + cl_policy.add_column(col_desc, key, cql_type) + cluster = Cluster(column_encryption_policy=cl_policy) + +:class:`~.AES256ColumnEncryptionPolicy` is a subclass of :class:`~.ColumnEncryptionPolicy` which provides +encryption and decryption via AES-256. This class is currently the only available column encryption policy +implementation, although users can certainly implement their own by subclassing :class:`~.ColumnEncryptionPolicy`. + +:class:`~.ColDesc` is a named tuple which uniquely identifies a column in a given keyspace and table. When we +have this tuple, the encryption key and the CQL type contained by this column we can add the column to the policy +using :func:`~.ColumnEncryptionPolicy.add_column`. Once we have added all column definitions to the policy we +pass it along to the cluster. + +The CQL type for the column only has meaning at the client; it is never sent to Cassandra. The encryption key +is also never sent to the server; all the server ever sees are random bytes reflecting the encrypted data. As a +result all columns containing client-side encrypted values should be declared with the CQL type "blob" at the +Cassandra server. + +Usage +----- +Client-side encryption shines most when used with prepared statements. A prepared statement is aware of information +about the columns in the query it was built from and we can use this information to transparently encrypt any +supplied parameters. For example, we can create a prepared statement to insert a value into column1 (as defined above) +by executing the following code after creating a :class:`~.Cluster` in the manner described above: + +.. code-block:: python + session = cluster.connect() + prepared = session.prepare("insert into ks1.table1 (column1) values (?)") + session.execute(prepared, (1000,)) + +Our encryption policy will detect that "column1" is an encrypted column and take appropriate action. + +Decryption of values returned from the server is also transparent. Whether we're executing a simple or prepared +statement encrypted columns will be decrypted automatically and made available via rows just like any other +result. + +Limitations +----------- +:class:`~.AES256ColumnEncryptionPolicy` uses the implementation of AES-256 provided by the +`cryptography `_ module. Any limitations of this module should be considered +when deploying client-side encryption. Note specifically that a Rust compiler is required for modern versions +of the cryptography package, although wheels exist for many common platforms. + +Client-side encryption has been implemented for both the default Cython and pure Python row processing logic. +This functionality has not yet been ported to the NumPy Cython implementation. We have reason to believe the +NumPy processing works reasonably well on Python 3.7 but fails for Python 3.8. We hope to address this discrepancy +in a future release. \ No newline at end of file diff --git a/docs/index.rst b/docs/index.rst index 978faa17c6..6eb943dda8 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -50,6 +50,9 @@ Contents :doc:`cloud` A guide to connecting to Datastax Astra +:doc:`column_encryption` + Transparent client-side per-column encryption and decryption + :doc:`geo_types` Working with DSE geometry types From fe0aebde2568264f48ae80c9837c2ef5be5b0aac Mon Sep 17 00:00:00 2001 From: Bret McGuire Date: Thu, 27 Apr 2023 01:22:09 -0500 Subject: [PATCH 3/5] Add docs for encode_and_encrypt --- cassandra/policies.py | 4 ++++ docs/column_encryption.rst | 17 ++++++++++++++++- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/cassandra/policies.py b/cassandra/policies.py index 1895a6f548..da8b57958a 100644 --- a/cassandra/policies.py +++ b/cassandra/policies.py @@ -1244,6 +1244,10 @@ def contains_column(self, coldesc): raise NotImplementedError() def encode_and_encrypt(self, coldesc, obj): + """ + Helper function to enable use of this policy on simple (i.e. non-prepared) + statements. + """ raise NotImplementedError() AES256_BLOCK_SIZE = 128 diff --git a/docs/column_encryption.rst b/docs/column_encryption.rst index ca591a0e3e..31ec31781b 100644 --- a/docs/column_encryption.rst +++ b/docs/column_encryption.rst @@ -48,6 +48,9 @@ Cassandra server. Usage ----- + +Encryption +^^^^^^^^^^ Client-side encryption shines most when used with prepared statements. A prepared statement is aware of information about the columns in the query it was built from and we can use this information to transparently encrypt any supplied parameters. For example, we can create a prepared statement to insert a value into column1 (as defined above) @@ -60,7 +63,19 @@ by executing the following code after creating a :class:`~.Cluster` in the manne Our encryption policy will detect that "column1" is an encrypted column and take appropriate action. -Decryption of values returned from the server is also transparent. Whether we're executing a simple or prepared +As mentioned above client-side encryption can also be used with simple queries, although such use cases are +certainly not transparent. :class:`~.ColumnEncryptionPolicy` provides a helper named +:func:`~.ColumnEncryptionPolicy.encode_and_encrypt` which will convert an input value into bytes using the +standard serialization methods employed by the driver. The result is then encrypted according to the configuration +of the policy. Using this approach the example above could be implemented along the lines of the following: + +.. code-block:: python + session = cluster.connect() + session.execute("insert into ks1.table1 (column1) values (%s)",(cl_policy.encode_and_encrypt(col_desc, 1000),)) + +Decryption +^^^^^^^^^^ +Decryption of values returned from the server is always transparent. Whether we're executing a simple or prepared statement encrypted columns will be decrypted automatically and made available via rows just like any other result. From a7ce0939d5b136cb78d56ca56b68ba14789c564f Mon Sep 17 00:00:00 2001 From: Bret McGuire Date: Thu, 27 Apr 2023 05:39:59 -0500 Subject: [PATCH 4/5] Fixes from code review --- cassandra/policies.py | 12 ++++++------ docs/column_encryption.rst | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/cassandra/policies.py b/cassandra/policies.py index da8b57958a..cd4a290f86 100644 --- a/cassandra/policies.py +++ b/cassandra/policies.py @@ -624,7 +624,7 @@ class ReconnectionPolicy(object): def new_schedule(self): """ This should return a finite or infinite iterable of delays (each as a - floating point number of seconds) inbetween each failed reconnection + floating point number of seconds) in-between each failed reconnection attempt. Note that if the iterable is finite, reconnection attempts will cease once the iterable is exhausted. """ @@ -634,12 +634,12 @@ def new_schedule(self): class ConstantReconnectionPolicy(ReconnectionPolicy): """ A :class:`.ReconnectionPolicy` subclass which sleeps for a fixed delay - inbetween each reconnection attempt. + in-between each reconnection attempt. """ def __init__(self, delay, max_attempts=64): """ - `delay` should be a floating point number of seconds to wait inbetween + `delay` should be a floating point number of seconds to wait in-between each attempt. `max_attempts` should be a total number of attempts to be made before @@ -663,7 +663,7 @@ def new_schedule(self): class ExponentialReconnectionPolicy(ReconnectionPolicy): """ A :class:`.ReconnectionPolicy` subclass which exponentially increases - the length of the delay inbetween each reconnection attempt up to + the length of the delay in-between each reconnection attempt up to a set maximum delay. A random amount of jitter (+/- 15%) will be added to the pure exponential @@ -723,7 +723,7 @@ class RetryPolicy(object): timeout and unavailable failures. These are failures reported from the server side. Timeouts are configured by `settings in cassandra.yaml `_. - Unavailable failures occur when the coordinator cannot acheive the consistency + Unavailable failures occur when the coordinator cannot achieve the consistency level for a request. For further information see the method descriptions below. @@ -1239,7 +1239,7 @@ def add_column(self, coldesc, key): def contains_column(self, coldesc): """ Predicate to determine if a specific column is supported by this policy. - Currently only used interally. + Currently only used internally. """ raise NotImplementedError() diff --git a/docs/column_encryption.rst b/docs/column_encryption.rst index 31ec31781b..4d2a6c2d91 100644 --- a/docs/column_encryption.rst +++ b/docs/column_encryption.rst @@ -12,7 +12,7 @@ also available, although in this case values must be manually encrypted and/or d :class:`~.ColumnEncryptionPolicy` instance provides methods to assist with these operations. Client-side encryption and decryption should work against all versions of Cassandra and DSE. It does not -utilize any server-side functionality to do it's work. +utilize any server-side functionality to do its work. Configuration ------------- From 1354974f87ee0cb51d7eecd0b72cf725d697cff6 Mon Sep 17 00:00:00 2001 From: Bret McGuire Date: Thu, 27 Apr 2023 16:59:20 -0500 Subject: [PATCH 5/5] Standardize on "falsy" over "falsey" --- cassandra/policies.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cassandra/policies.py b/cassandra/policies.py index cd4a290f86..26b9aa4c5a 100644 --- a/cassandra/policies.py +++ b/cassandra/policies.py @@ -463,7 +463,7 @@ class HostFilterPolicy(LoadBalancingPolicy): A :class:`.LoadBalancingPolicy` subclass configured with a child policy, and a single-argument predicate. This policy defers to the child policy for hosts where ``predicate(host)`` is truthy. Hosts for which - ``predicate(host)`` is falsey will be considered :attr:`.IGNORED`, and will + ``predicate(host)`` is falsy will be considered :attr:`.IGNORED`, and will not be used in a query plan. This can be used in the cases where you need a whitelist or blacklist @@ -499,7 +499,7 @@ def __init__(self, child_policy, predicate): :param child_policy: an instantiated :class:`.LoadBalancingPolicy` that this one will defer to. :param predicate: a one-parameter function that takes a :class:`.Host`. - If it returns a falsey value, the :class:`.Host` will + If it returns a falsy value, the :class:`.Host` will be :attr:`.IGNORED` and not returned in query plans. """ super(HostFilterPolicy, self).__init__() @@ -535,7 +535,7 @@ def predicate(self): def distance(self, host): """ Checks if ``predicate(host)``, then returns - :attr:`~HostDistance.IGNORED` if falsey, and defers to the child policy + :attr:`~HostDistance.IGNORED` if falsy, and defers to the child policy otherwise. """ if self.predicate(host):