From 1e4ed6d0d2ef68c3bb94ab63ccaa0d351bc31672 Mon Sep 17 00:00:00 2001 From: Gidon Gershinsky Date: Tue, 25 May 2021 15:57:51 +0300 Subject: [PATCH 01/11] native encryption paramateres --- .../encryption/NativeFileDecryption.java | 78 +++++++++++++ .../encryption/NativeFileEncryption.java | 103 ++++++++++++++++++ 2 files changed, 181 insertions(+) create mode 100644 core/src/main/java/org/apache/iceberg/encryption/NativeFileDecryption.java create mode 100644 core/src/main/java/org/apache/iceberg/encryption/NativeFileEncryption.java diff --git a/core/src/main/java/org/apache/iceberg/encryption/NativeFileDecryption.java b/core/src/main/java/org/apache/iceberg/encryption/NativeFileDecryption.java new file mode 100644 index 000000000000..3262d05dd21c --- /dev/null +++ b/core/src/main/java/org/apache/iceberg/encryption/NativeFileDecryption.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + + +package org.apache.iceberg.encryption; + +import java.nio.ByteBuffer; +import java.util.Map; + +/** + * The data keys and other parameters should be retrieved/unwrapped centrally (e.g., in a driver), by parsing the + * manifest key_metadata entry for a data file; and then sent to the worker that reads/decrypts this file in a native + * format. + * Key unwrapping requires authorization checks, and can involve interaction with a KMS. Therefore, unwrap only + * projected columns. + */ +public class NativeFileDecryption { + private ByteBuffer fileAadPrefix; + private Map fileDataKeys; + + private NativeFileDecryption(Map fileDataKeys, ByteBuffer fileAadPrefix) { + this.fileDataKeys = fileDataKeys; + this.fileAadPrefix = fileAadPrefix; + } + + /** + * Data encryption keys for a single file. + * NOTE: pass keys only for projected columns. + * @param dataKeys Map dekId -> dek. + * dekId is unique only within single file scope. + * dekIds are retrieved from manifest key_metadata field, along with the wrapped DEKs. + */ + public static Builder create(Map dataKeys) { + return new Builder(dataKeys); + } + + public static class Builder { + private ByteBuffer fileAadPrefix; + private Map fileDataKeys; + + private Builder(Map dataKeys) { + this.fileDataKeys = dataKeys; + } + + public Builder aadPrefix(ByteBuffer aadPrefix) { + this.fileAadPrefix = aadPrefix; + return this; + } + + public NativeFileDecryption build() { + return new NativeFileDecryption(fileDataKeys, fileAadPrefix); + } + } + + public ByteBuffer aadPrefix() { + return fileAadPrefix; + } + + public Map fileDataKeys() { + return fileDataKeys; + } +} diff --git a/core/src/main/java/org/apache/iceberg/encryption/NativeFileEncryption.java b/core/src/main/java/org/apache/iceberg/encryption/NativeFileEncryption.java new file mode 100644 index 000000000000..b772e863185c --- /dev/null +++ b/core/src/main/java/org/apache/iceberg/encryption/NativeFileEncryption.java @@ -0,0 +1,103 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + + +package org.apache.iceberg.encryption; + +import java.nio.ByteBuffer; +import java.util.Map; + +/** + * The data keys and other parameters should be generated centrally (e.g., in a driver). + * Each object (/set of keys) must be created for one data file only, and sent to the worker that writes/encrypts + * this file in a native format. + * The central process, that generates the data keys, will wrap them (encrypt with master keys) and store in the + * manifest key_metadata entry for the data file. Key wrapping can involve interaction with a KMS. + */ +public class NativeFileEncryption { + private ByteBuffer fileAadPrefix; + private Map fileDataKeys; + private String fileDekId; + private Map columnDekIds; + + private NativeFileEncryption(Map fileDataKeys, String fileDekId, + Map columnDekIds, ByteBuffer fileAadPrefix) { + // TODO check + this.fileDataKeys = fileDataKeys; + this.fileDekId = fileDekId; + this.columnDekIds = columnDekIds; + this.fileAadPrefix = fileAadPrefix; + } + + /** + * Data encryption keys for a single file. + * @param dataKeys Map dekId -> dek. + * dekId is unique only within single file scope, can be a simple counter. + * dekIds must be stored in manifest key_metadata field, along with the wrapped DEKs. + */ + public static Builder create(Map dataKeys) { + return new Builder(dataKeys); + } + + public static class Builder { + private ByteBuffer fileAadPrefix; + private Map fileDataKeys; + private String fileDekId; + private Map columnDekIds; + + private Builder(Map dataKeys) { + this.fileDataKeys = dataKeys; + } + + public Builder fileKeyId(String keyId) { + this.fileDekId = keyId; + return this; + } + + public Builder columnKeyIds(Map columnKeyIds) { + this.columnDekIds = columnKeyIds; + return this; + } + + public Builder aadPrefix(ByteBuffer aadPrefix) { + this.fileAadPrefix = aadPrefix; + return this; + } + + public NativeFileEncryption build() { + return new NativeFileEncryption(fileDataKeys, fileDekId, columnDekIds, fileAadPrefix); + } + } + + public ByteBuffer aadPrefix() { + return fileAadPrefix; + } + + public String fileDekId() { + return fileDekId; + } + + public Map fileDataKeys() { + return fileDataKeys; + } + + public Map columnDekIds() { + return columnDekIds; + } +} From 6fcc7d70bf8ca1c2dd7ed713b8181f884dbf8f99 Mon Sep 17 00:00:00 2001 From: Gidon Gershinsky Date: Wed, 26 May 2021 15:48:22 +0300 Subject: [PATCH 02/11] api refactor --- .../encryption/EncryptedInputFile.java | 12 ++++ .../encryption/EncryptedOutputFile.java | 12 ++++ .../encryption/NativeFileDecryptParams.java | 48 ++++++++++++++++ .../encryption/NativeFileEncryptParams.java | 55 +++++++++++++++++++ ....java => NativeFileDecryptParamsImpl.java} | 10 ++-- ....java => NativeFileEncryptParamsImpl.java} | 14 +++-- 6 files changed, 142 insertions(+), 9 deletions(-) create mode 100644 api/src/main/java/org/apache/iceberg/encryption/NativeFileDecryptParams.java create mode 100644 api/src/main/java/org/apache/iceberg/encryption/NativeFileEncryptParams.java rename core/src/main/java/org/apache/iceberg/encryption/{NativeFileDecryption.java => NativeFileDecryptParamsImpl.java} (87%) rename core/src/main/java/org/apache/iceberg/encryption/{NativeFileEncryption.java => NativeFileEncryptParamsImpl.java} (86%) diff --git a/api/src/main/java/org/apache/iceberg/encryption/EncryptedInputFile.java b/api/src/main/java/org/apache/iceberg/encryption/EncryptedInputFile.java index e990d1f5bf3a..36f306e473dd 100644 --- a/api/src/main/java/org/apache/iceberg/encryption/EncryptedInputFile.java +++ b/api/src/main/java/org/apache/iceberg/encryption/EncryptedInputFile.java @@ -21,6 +21,7 @@ import org.apache.iceberg.io.InputFile; +// TODO update class description /** * Thin wrapper around an {@link InputFile} instance that is encrypted. *

@@ -30,6 +31,13 @@ */ public interface EncryptedInputFile { + /** + * Use flat filestream decryption (default) or native format decryption + */ + default boolean useNativeEncryption() { + return false; + } + /** * The {@link InputFile} that is reading raw encrypted bytes from the underlying file system. */ @@ -40,4 +48,8 @@ public interface EncryptedInputFile { * by {@link #encryptedInputFile()}. */ EncryptionKeyMetadata keyMetadata(); + + default NativeFileDecryptParams nativeDecryptionParameters() { + return null; + } } diff --git a/api/src/main/java/org/apache/iceberg/encryption/EncryptedOutputFile.java b/api/src/main/java/org/apache/iceberg/encryption/EncryptedOutputFile.java index d05033ebe150..442648bc18e2 100644 --- a/api/src/main/java/org/apache/iceberg/encryption/EncryptedOutputFile.java +++ b/api/src/main/java/org/apache/iceberg/encryption/EncryptedOutputFile.java @@ -21,6 +21,7 @@ import org.apache.iceberg.io.OutputFile; +// TODO update class description /** * Thin wrapper around a {@link OutputFile} that is encrypting bytes written to the underlying * file system, via an encryption key that is symbolized by the enclosed @@ -31,6 +32,13 @@ */ public interface EncryptedOutputFile { + /** + * Use flat filestream encryption (default) or native format encryption + */ + default boolean useNativeEncryption() { + return false; + } + /** * An OutputFile instance that encrypts the bytes that are written to its output streams. */ @@ -41,4 +49,8 @@ public interface EncryptedOutputFile { * {@link #encryptingOutputFile()}. */ EncryptionKeyMetadata keyMetadata(); + + default NativeFileEncryptParams nativeEncryptionParameters() { + return null; + } } diff --git a/api/src/main/java/org/apache/iceberg/encryption/NativeFileDecryptParams.java b/api/src/main/java/org/apache/iceberg/encryption/NativeFileDecryptParams.java new file mode 100644 index 000000000000..15c145f2f2e3 --- /dev/null +++ b/api/src/main/java/org/apache/iceberg/encryption/NativeFileDecryptParams.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + + +package org.apache.iceberg.encryption; + +import java.io.Serializable; +import java.nio.ByteBuffer; +import java.util.Map; + +/** + * Per-data-file decryption parameters. + * The data keys and AADPrefix should be retrieved/unwrapped centrally (e.g., in a driver), by parsing the + * manifest key_metadata entry for a data file; and then sent to the worker that reads/decrypts this file in a native + * format. + * Key unwrapping requires authorization checks, and can involve interaction with a KMS. Therefore, unwrap only + * projected columns. + */ +public interface NativeFileDecryptParams extends Serializable { + + /** + * Data encryption keys for a single file. + * NOTE: pass keys only for projected columns. + * dataKeys Map dekId -> dek. + * dekId is unique only within single file scope. + * dekIds are retrieved from manifest key_metadata field, along with the wrapped DEKs. + */ + + Map fileDataKeys(); + + ByteBuffer aadPrefix(); +} diff --git a/api/src/main/java/org/apache/iceberg/encryption/NativeFileEncryptParams.java b/api/src/main/java/org/apache/iceberg/encryption/NativeFileEncryptParams.java new file mode 100644 index 000000000000..e29cec8b0fad --- /dev/null +++ b/api/src/main/java/org/apache/iceberg/encryption/NativeFileEncryptParams.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + + +package org.apache.iceberg.encryption; + +import java.io.Serializable; +import java.nio.ByteBuffer; +import java.util.Map; + +/** + * Per-data-file encryption parameters. + * The data keys and other parameters should be generated centrally (e.g., in a driver). + * Each object (/set of keys) must be created for one data file only, and sent to the worker that writes/encrypts + * this file in a native format. + * The central process, that generates the data keys, will wrap them (encrypt with master keys) and store in the + * manifest key_metadata entry for the data file. Key wrapping can involve interaction with a KMS. + */ +public interface NativeFileEncryptParams extends Serializable { + + /** + * Data encryption keys for a single file. + * dataKeys Map dekId -> dek. + * dekId is unique only within single file scope, can be a simple counter. + * dekIds must be stored in manifest key_metadata field, along with the wrapped DEKs. + */ + Map fileDataKeys(); + + ByteBuffer aadPrefix(); + + String fileDekId(); + + /** + * List of encrypted columns, each with its dek id + * columnDeks Map columnName -> dekId + * For nested columns, the name is a dot-separated string. + */ + Map columnDekIds(); +} diff --git a/core/src/main/java/org/apache/iceberg/encryption/NativeFileDecryption.java b/core/src/main/java/org/apache/iceberg/encryption/NativeFileDecryptParamsImpl.java similarity index 87% rename from core/src/main/java/org/apache/iceberg/encryption/NativeFileDecryption.java rename to core/src/main/java/org/apache/iceberg/encryption/NativeFileDecryptParamsImpl.java index 3262d05dd21c..2e9bae53f859 100644 --- a/core/src/main/java/org/apache/iceberg/encryption/NativeFileDecryption.java +++ b/core/src/main/java/org/apache/iceberg/encryption/NativeFileDecryptParamsImpl.java @@ -30,11 +30,11 @@ * Key unwrapping requires authorization checks, and can involve interaction with a KMS. Therefore, unwrap only * projected columns. */ -public class NativeFileDecryption { +public class NativeFileDecryptParamsImpl implements NativeFileDecryptParams { private ByteBuffer fileAadPrefix; private Map fileDataKeys; - private NativeFileDecryption(Map fileDataKeys, ByteBuffer fileAadPrefix) { + private NativeFileDecryptParamsImpl(Map fileDataKeys, ByteBuffer fileAadPrefix) { this.fileDataKeys = fileDataKeys; this.fileAadPrefix = fileAadPrefix; } @@ -63,15 +63,17 @@ public Builder aadPrefix(ByteBuffer aadPrefix) { return this; } - public NativeFileDecryption build() { - return new NativeFileDecryption(fileDataKeys, fileAadPrefix); + public NativeFileDecryptParamsImpl build() { + return new NativeFileDecryptParamsImpl(fileDataKeys, fileAadPrefix); } } + @Override public ByteBuffer aadPrefix() { return fileAadPrefix; } + @Override public Map fileDataKeys() { return fileDataKeys; } diff --git a/core/src/main/java/org/apache/iceberg/encryption/NativeFileEncryption.java b/core/src/main/java/org/apache/iceberg/encryption/NativeFileEncryptParamsImpl.java similarity index 86% rename from core/src/main/java/org/apache/iceberg/encryption/NativeFileEncryption.java rename to core/src/main/java/org/apache/iceberg/encryption/NativeFileEncryptParamsImpl.java index b772e863185c..6b7f287a90b2 100644 --- a/core/src/main/java/org/apache/iceberg/encryption/NativeFileEncryption.java +++ b/core/src/main/java/org/apache/iceberg/encryption/NativeFileEncryptParamsImpl.java @@ -30,14 +30,14 @@ * The central process, that generates the data keys, will wrap them (encrypt with master keys) and store in the * manifest key_metadata entry for the data file. Key wrapping can involve interaction with a KMS. */ -public class NativeFileEncryption { +public class NativeFileEncryptParamsImpl implements NativeFileEncryptParams { private ByteBuffer fileAadPrefix; private Map fileDataKeys; private String fileDekId; private Map columnDekIds; - private NativeFileEncryption(Map fileDataKeys, String fileDekId, - Map columnDekIds, ByteBuffer fileAadPrefix) { + private NativeFileEncryptParamsImpl(Map fileDataKeys, String fileDekId, + Map columnDekIds, ByteBuffer fileAadPrefix) { // TODO check this.fileDataKeys = fileDataKeys; this.fileDekId = fileDekId; @@ -80,23 +80,27 @@ public Builder aadPrefix(ByteBuffer aadPrefix) { return this; } - public NativeFileEncryption build() { - return new NativeFileEncryption(fileDataKeys, fileDekId, columnDekIds, fileAadPrefix); + public NativeFileEncryptParamsImpl build() { + return new NativeFileEncryptParamsImpl(fileDataKeys, fileDekId, columnDekIds, fileAadPrefix); } } + @Override public ByteBuffer aadPrefix() { return fileAadPrefix; } + @Override public String fileDekId() { return fileDekId; } + @Override public Map fileDataKeys() { return fileDataKeys; } + @Override public Map columnDekIds() { return columnDekIds; } From a950330fa7d5dec4d19d87aba75565140ae66f7a Mon Sep 17 00:00:00 2001 From: Gidon Gershinsky Date: Fri, 28 May 2021 09:11:54 +0300 Subject: [PATCH 03/11] fix javadoc errors --- .../apache/iceberg/encryption/EncryptedInputFile.java | 2 +- .../iceberg/encryption/NativeFileDecryptParams.java | 7 +++---- .../iceberg/encryption/NativeFileEncryptParams.java | 10 +++++----- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/api/src/main/java/org/apache/iceberg/encryption/EncryptedInputFile.java b/api/src/main/java/org/apache/iceberg/encryption/EncryptedInputFile.java index 36f306e473dd..bf3a84a6d2b4 100644 --- a/api/src/main/java/org/apache/iceberg/encryption/EncryptedInputFile.java +++ b/api/src/main/java/org/apache/iceberg/encryption/EncryptedInputFile.java @@ -34,7 +34,7 @@ public interface EncryptedInputFile { /** * Use flat filestream decryption (default) or native format decryption */ - default boolean useNativeEncryption() { + default boolean useNativeDecryption() { return false; } diff --git a/api/src/main/java/org/apache/iceberg/encryption/NativeFileDecryptParams.java b/api/src/main/java/org/apache/iceberg/encryption/NativeFileDecryptParams.java index 15c145f2f2e3..59a7f3b486dc 100644 --- a/api/src/main/java/org/apache/iceberg/encryption/NativeFileDecryptParams.java +++ b/api/src/main/java/org/apache/iceberg/encryption/NativeFileDecryptParams.java @@ -37,11 +37,10 @@ public interface NativeFileDecryptParams extends Serializable { /** * Data encryption keys for a single file. * NOTE: pass keys only for projected columns. - * dataKeys Map dekId -> dek. - * dekId is unique only within single file scope. - * dekIds are retrieved from manifest key_metadata field, along with the wrapped DEKs. + * Passed as a Map (dekId to dek). + * dekId is unique only within single file scope. + * dekIds are retrieved from manifest key_metadata field, along with the wrapped DEKs. */ - Map fileDataKeys(); ByteBuffer aadPrefix(); diff --git a/api/src/main/java/org/apache/iceberg/encryption/NativeFileEncryptParams.java b/api/src/main/java/org/apache/iceberg/encryption/NativeFileEncryptParams.java index e29cec8b0fad..5e67c66415b9 100644 --- a/api/src/main/java/org/apache/iceberg/encryption/NativeFileEncryptParams.java +++ b/api/src/main/java/org/apache/iceberg/encryption/NativeFileEncryptParams.java @@ -36,9 +36,9 @@ public interface NativeFileEncryptParams extends Serializable { /** * Data encryption keys for a single file. - * dataKeys Map dekId -> dek. - * dekId is unique only within single file scope, can be a simple counter. - * dekIds must be stored in manifest key_metadata field, along with the wrapped DEKs. + * Passed as a Map (dekId to dek). + * dekId is unique only within single file scope, can be a simple counter. + * dekIds must be stored in manifest key_metadata field, along with the wrapped DEKs. */ Map fileDataKeys(); @@ -47,8 +47,8 @@ public interface NativeFileEncryptParams extends Serializable { String fileDekId(); /** - * List of encrypted columns, each with its dek id - * columnDeks Map columnName -> dekId + * List of encrypted columns, each with its dek id. + * Passed as a Map (columnName to dekId) * For nested columns, the name is a dot-separated string. */ Map columnDekIds(); From 00ce5375fb27c28cb84f252cd6b5ffba4f759db1 Mon Sep 17 00:00:00 2001 From: Gidon Gershinsky Date: Thu, 21 Oct 2021 14:00:55 +0300 Subject: [PATCH 04/11] simplify and update to MVP version --- .../NativeFileCryptoParameters.java | 94 +++++++++++++++ .../encryption/NativeFileDecryptParams.java | 47 -------- .../encryption/NativeFileEncryptParams.java | 55 --------- .../NativeFileDecryptParamsImpl.java | 80 ------------- .../NativeFileEncryptParamsImpl.java | 107 ------------------ 5 files changed, 94 insertions(+), 289 deletions(-) create mode 100644 api/src/main/java/org/apache/iceberg/encryption/NativeFileCryptoParameters.java delete mode 100644 api/src/main/java/org/apache/iceberg/encryption/NativeFileDecryptParams.java delete mode 100644 api/src/main/java/org/apache/iceberg/encryption/NativeFileEncryptParams.java delete mode 100644 core/src/main/java/org/apache/iceberg/encryption/NativeFileDecryptParamsImpl.java delete mode 100644 core/src/main/java/org/apache/iceberg/encryption/NativeFileEncryptParamsImpl.java diff --git a/api/src/main/java/org/apache/iceberg/encryption/NativeFileCryptoParameters.java b/api/src/main/java/org/apache/iceberg/encryption/NativeFileCryptoParameters.java new file mode 100644 index 000000000000..d8859f6ba77d --- /dev/null +++ b/api/src/main/java/org/apache/iceberg/encryption/NativeFileCryptoParameters.java @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + + +package org.apache.iceberg.encryption; + +import java.nio.ByteBuffer; +import java.util.Map; +import org.apache.iceberg.relocated.com.google.common.base.Preconditions; + +/** + * Barebone encryption parameters, one object per content file. + * Carries the file and column encryption keys (and optional AAD prefix). + */ +public class NativeFileCryptoParameters { + private ByteBuffer fileAadPrefix; + private Map columnKeys; + private ByteBuffer fileKey; + + private NativeFileCryptoParameters(Map columnKeys, ByteBuffer fileKey, + ByteBuffer fileAadPrefix) { + Preconditions.checkState((columnKeys != null && columnKeys.size() > 0) || fileKey != null, + "No file or column keys are supplied"); + this.columnKeys = columnKeys; + this.fileKey = fileKey; + this.fileAadPrefix = fileAadPrefix; + } + + /** + * Creates the builder. + * @param fileKey per-file encryption key. For example, used as "footer key" DEK in Parquet encryption. + */ + public static Builder create(ByteBuffer fileKey) { + return new Builder(fileKey); + } + + public static class Builder { + private ByteBuffer fileAadPrefix; + private Map columnKeys; + private ByteBuffer fileKey; + + private Builder(ByteBuffer fileKey) { + this.fileKey = fileKey; + } + + /** + * Set column encryption keys. + * @param columnKeyMap Map of column names to column keys. Column names must be the original names, + * used during content file creation. For example, Parquet will use them to find and + * encrypt the relevant columns. + */ + public Builder columnKeys(Map columnKeyMap) { + this.columnKeys = columnKeyMap; + return this; + } + + public Builder aadPrefix(ByteBuffer aadPrefix) { + this.fileAadPrefix = aadPrefix; + return this; + } + + public NativeFileCryptoParameters build() { + return new NativeFileCryptoParameters(columnKeys, fileKey, fileAadPrefix); + } + } + + public ByteBuffer aadPrefix() { + return fileAadPrefix; + } + + public ByteBuffer fileKey() { + return fileKey; + } + + public Map columnKeys() { + return columnKeys; + } +} diff --git a/api/src/main/java/org/apache/iceberg/encryption/NativeFileDecryptParams.java b/api/src/main/java/org/apache/iceberg/encryption/NativeFileDecryptParams.java deleted file mode 100644 index 59a7f3b486dc..000000000000 --- a/api/src/main/java/org/apache/iceberg/encryption/NativeFileDecryptParams.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - - -package org.apache.iceberg.encryption; - -import java.io.Serializable; -import java.nio.ByteBuffer; -import java.util.Map; - -/** - * Per-data-file decryption parameters. - * The data keys and AADPrefix should be retrieved/unwrapped centrally (e.g., in a driver), by parsing the - * manifest key_metadata entry for a data file; and then sent to the worker that reads/decrypts this file in a native - * format. - * Key unwrapping requires authorization checks, and can involve interaction with a KMS. Therefore, unwrap only - * projected columns. - */ -public interface NativeFileDecryptParams extends Serializable { - - /** - * Data encryption keys for a single file. - * NOTE: pass keys only for projected columns. - * Passed as a Map (dekId to dek). - * dekId is unique only within single file scope. - * dekIds are retrieved from manifest key_metadata field, along with the wrapped DEKs. - */ - Map fileDataKeys(); - - ByteBuffer aadPrefix(); -} diff --git a/api/src/main/java/org/apache/iceberg/encryption/NativeFileEncryptParams.java b/api/src/main/java/org/apache/iceberg/encryption/NativeFileEncryptParams.java deleted file mode 100644 index 5e67c66415b9..000000000000 --- a/api/src/main/java/org/apache/iceberg/encryption/NativeFileEncryptParams.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - - -package org.apache.iceberg.encryption; - -import java.io.Serializable; -import java.nio.ByteBuffer; -import java.util.Map; - -/** - * Per-data-file encryption parameters. - * The data keys and other parameters should be generated centrally (e.g., in a driver). - * Each object (/set of keys) must be created for one data file only, and sent to the worker that writes/encrypts - * this file in a native format. - * The central process, that generates the data keys, will wrap them (encrypt with master keys) and store in the - * manifest key_metadata entry for the data file. Key wrapping can involve interaction with a KMS. - */ -public interface NativeFileEncryptParams extends Serializable { - - /** - * Data encryption keys for a single file. - * Passed as a Map (dekId to dek). - * dekId is unique only within single file scope, can be a simple counter. - * dekIds must be stored in manifest key_metadata field, along with the wrapped DEKs. - */ - Map fileDataKeys(); - - ByteBuffer aadPrefix(); - - String fileDekId(); - - /** - * List of encrypted columns, each with its dek id. - * Passed as a Map (columnName to dekId) - * For nested columns, the name is a dot-separated string. - */ - Map columnDekIds(); -} diff --git a/core/src/main/java/org/apache/iceberg/encryption/NativeFileDecryptParamsImpl.java b/core/src/main/java/org/apache/iceberg/encryption/NativeFileDecryptParamsImpl.java deleted file mode 100644 index 2e9bae53f859..000000000000 --- a/core/src/main/java/org/apache/iceberg/encryption/NativeFileDecryptParamsImpl.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - - -package org.apache.iceberg.encryption; - -import java.nio.ByteBuffer; -import java.util.Map; - -/** - * The data keys and other parameters should be retrieved/unwrapped centrally (e.g., in a driver), by parsing the - * manifest key_metadata entry for a data file; and then sent to the worker that reads/decrypts this file in a native - * format. - * Key unwrapping requires authorization checks, and can involve interaction with a KMS. Therefore, unwrap only - * projected columns. - */ -public class NativeFileDecryptParamsImpl implements NativeFileDecryptParams { - private ByteBuffer fileAadPrefix; - private Map fileDataKeys; - - private NativeFileDecryptParamsImpl(Map fileDataKeys, ByteBuffer fileAadPrefix) { - this.fileDataKeys = fileDataKeys; - this.fileAadPrefix = fileAadPrefix; - } - - /** - * Data encryption keys for a single file. - * NOTE: pass keys only for projected columns. - * @param dataKeys Map dekId -> dek. - * dekId is unique only within single file scope. - * dekIds are retrieved from manifest key_metadata field, along with the wrapped DEKs. - */ - public static Builder create(Map dataKeys) { - return new Builder(dataKeys); - } - - public static class Builder { - private ByteBuffer fileAadPrefix; - private Map fileDataKeys; - - private Builder(Map dataKeys) { - this.fileDataKeys = dataKeys; - } - - public Builder aadPrefix(ByteBuffer aadPrefix) { - this.fileAadPrefix = aadPrefix; - return this; - } - - public NativeFileDecryptParamsImpl build() { - return new NativeFileDecryptParamsImpl(fileDataKeys, fileAadPrefix); - } - } - - @Override - public ByteBuffer aadPrefix() { - return fileAadPrefix; - } - - @Override - public Map fileDataKeys() { - return fileDataKeys; - } -} diff --git a/core/src/main/java/org/apache/iceberg/encryption/NativeFileEncryptParamsImpl.java b/core/src/main/java/org/apache/iceberg/encryption/NativeFileEncryptParamsImpl.java deleted file mode 100644 index 6b7f287a90b2..000000000000 --- a/core/src/main/java/org/apache/iceberg/encryption/NativeFileEncryptParamsImpl.java +++ /dev/null @@ -1,107 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - - -package org.apache.iceberg.encryption; - -import java.nio.ByteBuffer; -import java.util.Map; - -/** - * The data keys and other parameters should be generated centrally (e.g., in a driver). - * Each object (/set of keys) must be created for one data file only, and sent to the worker that writes/encrypts - * this file in a native format. - * The central process, that generates the data keys, will wrap them (encrypt with master keys) and store in the - * manifest key_metadata entry for the data file. Key wrapping can involve interaction with a KMS. - */ -public class NativeFileEncryptParamsImpl implements NativeFileEncryptParams { - private ByteBuffer fileAadPrefix; - private Map fileDataKeys; - private String fileDekId; - private Map columnDekIds; - - private NativeFileEncryptParamsImpl(Map fileDataKeys, String fileDekId, - Map columnDekIds, ByteBuffer fileAadPrefix) { - // TODO check - this.fileDataKeys = fileDataKeys; - this.fileDekId = fileDekId; - this.columnDekIds = columnDekIds; - this.fileAadPrefix = fileAadPrefix; - } - - /** - * Data encryption keys for a single file. - * @param dataKeys Map dekId -> dek. - * dekId is unique only within single file scope, can be a simple counter. - * dekIds must be stored in manifest key_metadata field, along with the wrapped DEKs. - */ - public static Builder create(Map dataKeys) { - return new Builder(dataKeys); - } - - public static class Builder { - private ByteBuffer fileAadPrefix; - private Map fileDataKeys; - private String fileDekId; - private Map columnDekIds; - - private Builder(Map dataKeys) { - this.fileDataKeys = dataKeys; - } - - public Builder fileKeyId(String keyId) { - this.fileDekId = keyId; - return this; - } - - public Builder columnKeyIds(Map columnKeyIds) { - this.columnDekIds = columnKeyIds; - return this; - } - - public Builder aadPrefix(ByteBuffer aadPrefix) { - this.fileAadPrefix = aadPrefix; - return this; - } - - public NativeFileEncryptParamsImpl build() { - return new NativeFileEncryptParamsImpl(fileDataKeys, fileDekId, columnDekIds, fileAadPrefix); - } - } - - @Override - public ByteBuffer aadPrefix() { - return fileAadPrefix; - } - - @Override - public String fileDekId() { - return fileDekId; - } - - @Override - public Map fileDataKeys() { - return fileDataKeys; - } - - @Override - public Map columnDekIds() { - return columnDekIds; - } -} From 82b8d94cd0703535df5199c7349a566d050fa046 Mon Sep 17 00:00:00 2001 From: Gidon Gershinsky Date: Thu, 4 Nov 2021 15:59:15 +0200 Subject: [PATCH 05/11] updates after review round 1 --- .../encryption/EncryptedInputFile.java | 12 -- .../encryption/EncryptedOutputFile.java | 9 +- .../iceberg/encryption/EncryptionManager.java | 18 +++ .../NativeFileCryptoParameters.java | 16 ++- .../org/apache/iceberg/TableProperties.java | 42 ++++++ .../apache/iceberg/encryption/Ciphers.java | 121 ++++++++++++++++++ .../encryption/EncryptionAlgorithm.java | 52 ++++++++ .../encryption/NativeEncryptedInputFile.java | 63 +++++++++ .../encryption/NativeEncryptedOutputFile.java | 56 ++++++++ .../apache/iceberg/io/WrappedInputStream.java | 68 ++++++++++ 10 files changed, 440 insertions(+), 17 deletions(-) create mode 100644 core/src/main/java/org/apache/iceberg/encryption/Ciphers.java create mode 100644 core/src/main/java/org/apache/iceberg/encryption/EncryptionAlgorithm.java create mode 100644 core/src/main/java/org/apache/iceberg/encryption/NativeEncryptedInputFile.java create mode 100644 core/src/main/java/org/apache/iceberg/encryption/NativeEncryptedOutputFile.java create mode 100644 core/src/main/java/org/apache/iceberg/io/WrappedInputStream.java diff --git a/api/src/main/java/org/apache/iceberg/encryption/EncryptedInputFile.java b/api/src/main/java/org/apache/iceberg/encryption/EncryptedInputFile.java index bf3a84a6d2b4..e990d1f5bf3a 100644 --- a/api/src/main/java/org/apache/iceberg/encryption/EncryptedInputFile.java +++ b/api/src/main/java/org/apache/iceberg/encryption/EncryptedInputFile.java @@ -21,7 +21,6 @@ import org.apache.iceberg.io.InputFile; -// TODO update class description /** * Thin wrapper around an {@link InputFile} instance that is encrypted. *

@@ -31,13 +30,6 @@ */ public interface EncryptedInputFile { - /** - * Use flat filestream decryption (default) or native format decryption - */ - default boolean useNativeDecryption() { - return false; - } - /** * The {@link InputFile} that is reading raw encrypted bytes from the underlying file system. */ @@ -48,8 +40,4 @@ default boolean useNativeDecryption() { * by {@link #encryptedInputFile()}. */ EncryptionKeyMetadata keyMetadata(); - - default NativeFileDecryptParams nativeDecryptionParameters() { - return null; - } } diff --git a/api/src/main/java/org/apache/iceberg/encryption/EncryptedOutputFile.java b/api/src/main/java/org/apache/iceberg/encryption/EncryptedOutputFile.java index 442648bc18e2..ddf8d7d0b6f8 100644 --- a/api/src/main/java/org/apache/iceberg/encryption/EncryptedOutputFile.java +++ b/api/src/main/java/org/apache/iceberg/encryption/EncryptedOutputFile.java @@ -21,8 +21,8 @@ import org.apache.iceberg.io.OutputFile; -// TODO update class description /** + * TODO update comment * Thin wrapper around a {@link OutputFile} that is encrypting bytes written to the underlying * file system, via an encryption key that is symbolized by the enclosed * {@link EncryptionKeyMetadata}. @@ -33,7 +33,7 @@ public interface EncryptedOutputFile { /** - * Use flat filestream encryption (default) or native format encryption + * Use flat filestream encryption (default) or pushdown to native format encryption */ default boolean useNativeEncryption() { return false; @@ -50,7 +50,10 @@ default boolean useNativeEncryption() { */ EncryptionKeyMetadata keyMetadata(); - default NativeFileEncryptParams nativeEncryptionParameters() { + /** + * Return parameters of native encryption (if the latter is used for this file) + */ + default NativeFileCryptoParameters nativeEncryptionParameters() { return null; } } diff --git a/api/src/main/java/org/apache/iceberg/encryption/EncryptionManager.java b/api/src/main/java/org/apache/iceberg/encryption/EncryptionManager.java index 97f6f639311b..2ba684723e84 100644 --- a/api/src/main/java/org/apache/iceberg/encryption/EncryptionManager.java +++ b/api/src/main/java/org/apache/iceberg/encryption/EncryptionManager.java @@ -71,4 +71,22 @@ default Iterable decrypt(Iterable encrypted) { default Iterable encrypt(Iterable rawOutput) { return Iterables.transform(rawOutput, this::encrypt); } + + /** + * Encrypt a manifest list + * @param rawOutput raw output file + * @return encrypted output file + */ + default EncryptedOutputFile encryptManifestList(OutputFile rawOutput) { + return encrypt(rawOutput); + } + + /** + * Encrypt a manifest file + * @param rawOutput raw output file + * @return encrypted output file + */ + default EncryptedOutputFile encryptManifestFile(OutputFile rawOutput) { + return encrypt(rawOutput); + } } diff --git a/api/src/main/java/org/apache/iceberg/encryption/NativeFileCryptoParameters.java b/api/src/main/java/org/apache/iceberg/encryption/NativeFileCryptoParameters.java index d8859f6ba77d..55d16c7c74bc 100644 --- a/api/src/main/java/org/apache/iceberg/encryption/NativeFileCryptoParameters.java +++ b/api/src/main/java/org/apache/iceberg/encryption/NativeFileCryptoParameters.java @@ -32,14 +32,16 @@ public class NativeFileCryptoParameters { private ByteBuffer fileAadPrefix; private Map columnKeys; private ByteBuffer fileKey; + private String fileEncryptionAlgorithm; private NativeFileCryptoParameters(Map columnKeys, ByteBuffer fileKey, - ByteBuffer fileAadPrefix) { + ByteBuffer fileAadPrefix, String fileEncryptionAlgorithm) { Preconditions.checkState((columnKeys != null && columnKeys.size() > 0) || fileKey != null, "No file or column keys are supplied"); this.columnKeys = columnKeys; this.fileKey = fileKey; this.fileAadPrefix = fileAadPrefix; + this.fileEncryptionAlgorithm = fileEncryptionAlgorithm; } /** @@ -54,6 +56,7 @@ public static class Builder { private ByteBuffer fileAadPrefix; private Map columnKeys; private ByteBuffer fileKey; + private String fileEncryptionAlgorithm; private Builder(ByteBuffer fileKey) { this.fileKey = fileKey; @@ -75,8 +78,13 @@ public Builder aadPrefix(ByteBuffer aadPrefix) { return this; } + public Builder encryptionAlgorithm(String encryptionAlgorithm) { + this.fileEncryptionAlgorithm = encryptionAlgorithm; + return this; + } + public NativeFileCryptoParameters build() { - return new NativeFileCryptoParameters(columnKeys, fileKey, fileAadPrefix); + return new NativeFileCryptoParameters(columnKeys, fileKey, fileAadPrefix, fileEncryptionAlgorithm); } } @@ -91,4 +99,8 @@ public ByteBuffer fileKey() { public Map columnKeys() { return columnKeys; } + + public String encryptionAlgorithm() { + return fileEncryptionAlgorithm; + } } diff --git a/core/src/main/java/org/apache/iceberg/TableProperties.java b/core/src/main/java/org/apache/iceberg/TableProperties.java index b9b54e76cb5d..43c3cc72ed77 100644 --- a/core/src/main/java/org/apache/iceberg/TableProperties.java +++ b/core/src/main/java/org/apache/iceberg/TableProperties.java @@ -276,8 +276,50 @@ private TableProperties() { @Deprecated public static final boolean MERGE_CARDINALITY_CHECK_ENABLED_DEFAULT = true; +<<<<<<< HEAD public static final String MERGE_DISTRIBUTION_MODE = "write.merge.distribution-mode"; public static final String UPSERT_ENABLED = "write.upsert.enabled"; public static final boolean UPSERT_ENABLED_DEFAULT = false; +======= + /** + * Encryption manager type + */ + public static final String ENCRYPTION_MANAGER_TYPE = "encryption.manager.type"; + public static final String ENCRYPTION_MANAGER_TYPE_PLAINTEXT = "plaintext"; + public static final String ENCRYPTION_MANAGER_TYPE_LEGACY = "legacy"; // TODO needed? + public static final String ENCRYPTION_MANAGER_TYPE_SINGLE_ENVELOPE = "envelope"; + public static final String ENCRYPTION_MANAGER_TYPE_DOUBLE_ENVELOPE = "double.envelope"; + + public static final String ENCRYPTION_TABLE_KEY = "encryption.table.key"; + + public static final String ENCRYPTION_DEK_LENGTH = "encryption.data.key.length"; + public static final int ENCRYPTION_DEK_LENGTH_DEFAULT = 16; + + public static final String ENCRYPTION_DATA_ALGORITHM = "encryption.data.algorithm"; + public static final String ENCRYPTION_DATA_ALGORITHM_DEFAULT = EncryptionAlgorithm.AES_GCM.toString(); + + /** + * Leverage file format native encryption instead of encrypting the entire file through Iceberg encryption stream + */ + public static final String ENCRYPTION_PUSHDOWN_ENABLED = "encryption.pushdown"; + public static final boolean ENCRYPTION_PUSHDOWN_ENABLED_DEFAULT = true; + + /** + * Implementation of the KMS client for envelope encryption. + */ + public static final String ENCRYPTION_KMS_CLIENT_IMPL = "encryption.kms.client-impl"; + + /** + * Implementation of custom out/input files (for metadata and Avro data) in envelope encryption. + */ + public static final String ENCRYPTION_OUTPUT_FILE_IMPL = "encryption.output.file-impl"; + public static final String ENCRYPTION_INPUT_FILE_IMPL = "encryption.input.file-impl"; + + /** + * Implementation of legacy encryption manager. + * TODO needed? + */ + public static final String ENCRYPTION_LEGACY_MANAGER_IMPL = "encryption.legacy.manager-impl"; +>>>>>>> d4aad3190 (updates after review round 1) } diff --git a/core/src/main/java/org/apache/iceberg/encryption/Ciphers.java b/core/src/main/java/org/apache/iceberg/encryption/Ciphers.java new file mode 100644 index 000000000000..4a2cc772bd4c --- /dev/null +++ b/core/src/main/java/org/apache/iceberg/encryption/Ciphers.java @@ -0,0 +1,121 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iceberg.encryption; + +import java.security.GeneralSecurityException; +import java.security.SecureRandom; +import javax.crypto.AEADBadTagException; +import javax.crypto.Cipher; +import javax.crypto.spec.GCMParameterSpec; +import javax.crypto.spec.SecretKeySpec; + +public class Ciphers { + + public static class AesGcmEncryptor { + public static final int NONCE_LENGTH = 12; + public static final int GCM_TAG_LENGTH = 16; + public static final int GCM_TAG_LENGTH_BITS = 8 * GCM_TAG_LENGTH; + + private final SecretKeySpec aesKey; + private final Cipher cipher; + private final SecureRandom randomGenerator; + + AesGcmEncryptor(byte[] keyBytes) { + int keyLength = keyBytes.length; + if (!(keyLength == 16 || keyLength == 24 || keyLength == 32)) { + throw new IllegalArgumentException("Wrong key length " + keyLength); + } + this.aesKey = new SecretKeySpec(keyBytes, "AES"); + + try { + this.cipher = Cipher.getInstance("AES/GCM/NoPadding"); + } catch (GeneralSecurityException e) { + throw new RuntimeException("Failed to create GCM cipher", e); + } + + this.randomGenerator = new SecureRandom(); + } + + public byte[] encrypt(byte[] plainText) { + byte[] nonce = new byte[NONCE_LENGTH]; + randomGenerator.nextBytes(nonce); + int cipherTextLength = NONCE_LENGTH + plainText.length + GCM_TAG_LENGTH; + byte[] cipherText = new byte[cipherTextLength]; + + try { + GCMParameterSpec spec = new GCMParameterSpec(GCM_TAG_LENGTH_BITS, nonce); + cipher.init(Cipher.ENCRYPT_MODE, aesKey, spec); + cipher.doFinal(plainText, 0, plainText.length, cipherText, NONCE_LENGTH); + } catch (GeneralSecurityException e) { + throw new RuntimeException("Failed to encrypt", e); + } + + // Add the nonce + System.arraycopy(nonce, 0, cipherText, 0, NONCE_LENGTH); + + return cipherText; + } + } + + public static class AesGcmDecryptor { + private final SecretKeySpec aesKey; + private final Cipher cipher; + + AesGcmDecryptor(byte[] keyBytes) { + int keyLength = keyBytes.length; + if (!(keyLength == 16 || keyLength == 24 || keyLength == 32)) { + throw new IllegalArgumentException("Wrong key length " + keyLength); + } + + this.aesKey = new SecretKeySpec(keyBytes, "AES"); + + try { + this.cipher = Cipher.getInstance("AES/GCM/NoPadding"); + } catch (GeneralSecurityException e) { + throw new RuntimeException("Failed to create GCM cipher", e); + } + } + + public byte[] decrypt(byte[] ciphertext) { + int plainTextLength = ciphertext.length - AesGcmEncryptor.GCM_TAG_LENGTH - AesGcmEncryptor.NONCE_LENGTH; + if (plainTextLength < 1) { + throw new RuntimeException("Wrong input length " + plainTextLength); + } + + // Get the nonce from ciphertext + byte[] nonce = new byte[AesGcmEncryptor.NONCE_LENGTH]; + System.arraycopy(ciphertext, 0, nonce, 0, AesGcmEncryptor.NONCE_LENGTH); + + byte[] plainText = new byte[plainTextLength]; + int inputLength = ciphertext.length - AesGcmEncryptor.NONCE_LENGTH; + try { + GCMParameterSpec spec = new GCMParameterSpec(AesGcmEncryptor.GCM_TAG_LENGTH_BITS, nonce); + cipher.init(Cipher.DECRYPT_MODE, aesKey, spec); + cipher.doFinal(ciphertext, AesGcmEncryptor.NONCE_LENGTH, inputLength, plainText, 0); + } catch (AEADBadTagException e) { + throw new RuntimeException("GCM tag check failed", e); + } catch (GeneralSecurityException e) { + throw new RuntimeException("Failed to decrypt", e); + } + + return plainText; + } + } +} diff --git a/core/src/main/java/org/apache/iceberg/encryption/EncryptionAlgorithm.java b/core/src/main/java/org/apache/iceberg/encryption/EncryptionAlgorithm.java new file mode 100644 index 000000000000..0ada5900e4b6 --- /dev/null +++ b/core/src/main/java/org/apache/iceberg/encryption/EncryptionAlgorithm.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iceberg.encryption; + +/** + * Algorithm supported for file encryption. + */ +public enum EncryptionAlgorithm { + /** + * Counter mode (CTR) allows fast encryption with high throughput. + * It is an encryption only cipher and does not ensure content integrity. + * Inputs to CTR cipher are: + * 1. encryption key + * 2. a 16-byte initialization vector (12-byte nonce, 4-byte counter) + * 3. plaintext data + */ + AES_CTR, + /** + * Galois/Counter mode (GCM) combines CTR with the new Galois mode of authentication. + * It not only ensures data confidentiality, but also ensures data integrity. + * Inputs to GCM cipher are: + * 1. encryption key + * 2. a 12-byte initialization vector + * 3. additional authenticated data + * 4. plaintext data + */ + AES_GCM, + /** + * A combination of GCM and CTR that can be used for file types like Parquet, + * so that all modules except pages are encrypted by GCM to ensure integrity, + * and CTR is used for efficient encryption of bulk data. + * The tradeoff is that attackers would be able to tamper page data. + */ + AES_GCM_CTR +} diff --git a/core/src/main/java/org/apache/iceberg/encryption/NativeEncryptedInputFile.java b/core/src/main/java/org/apache/iceberg/encryption/NativeEncryptedInputFile.java new file mode 100644 index 000000000000..178c35db29c6 --- /dev/null +++ b/core/src/main/java/org/apache/iceberg/encryption/NativeEncryptedInputFile.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iceberg.encryption; + +import org.apache.iceberg.io.InputFile; +import org.apache.iceberg.io.SeekableInputStream; +import org.apache.iceberg.io.WrappedInputStream; + +public class NativeEncryptedInputFile implements InputFile { + + private final InputFile rawInput; + private final NativeFileCryptoParameters nativeDecryptionParameteres; + + NativeEncryptedInputFile(InputFile rawInput, NativeFileCryptoParameters nativeDecryptionParameteres) { + this.rawInput = rawInput; + this.nativeDecryptionParameteres = nativeDecryptionParameteres; + } + + public NativeFileCryptoParameters nativeDecryptionParameters() { + return nativeDecryptionParameteres; + } + + @Override + public long getLength() { + return rawInput.getLength(); + } + + @Override + public SeekableInputStream newStream() { + // TODO remove this comment after review + // This class is not HadoopInputFile, while its rawInput can be; and rawInput's stream can be FSDataInputStream. + // Returning rawInput.newStream() here leads to closed stream exceptions, due to Hadoop handling in ParquetIO class. + // Therefore, using stream wrap. + return new WrappedInputStream(rawInput.newStream()); + } + + @Override + public String location() { + return rawInput.location(); + } + + @Override + public boolean exists() { + return rawInput.exists(); + } +} diff --git a/core/src/main/java/org/apache/iceberg/encryption/NativeEncryptedOutputFile.java b/core/src/main/java/org/apache/iceberg/encryption/NativeEncryptedOutputFile.java new file mode 100644 index 000000000000..7ddd489cefbf --- /dev/null +++ b/core/src/main/java/org/apache/iceberg/encryption/NativeEncryptedOutputFile.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iceberg.encryption; + +import org.apache.iceberg.io.OutputFile; + +class NativeEncryptedOutputFile implements EncryptedOutputFile { + + private final OutputFile rawOutput; + private final NativeFileCryptoParameters nativeEncryptionParameters; + private final EncryptionKeyMetadata keyMetadata; + + NativeEncryptedOutputFile(OutputFile rawOutput, EncryptionKeyMetadata keyMetadata, + NativeFileCryptoParameters nativeEncryptionParameters) { + this.rawOutput = rawOutput; + this.nativeEncryptionParameters = nativeEncryptionParameters; + this.keyMetadata = keyMetadata; + } + + @Override + public OutputFile encryptingOutputFile() { + return rawOutput; + } + + @Override + public EncryptionKeyMetadata keyMetadata() { + return keyMetadata; + } + + @Override + public NativeFileCryptoParameters nativeEncryptionParameters() { + return nativeEncryptionParameters; + } + + @Override + public boolean useNativeEncryption() { + return true; + } +} diff --git a/core/src/main/java/org/apache/iceberg/io/WrappedInputStream.java b/core/src/main/java/org/apache/iceberg/io/WrappedInputStream.java new file mode 100644 index 000000000000..42fe90a1ac6e --- /dev/null +++ b/core/src/main/java/org/apache/iceberg/io/WrappedInputStream.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iceberg.io; + +import java.io.IOException; + +/** + * Wrapping InputFiles can have HadoopInputFile as rawInput. Therefore, rawInput's stream can be + * FSDataInputStream. This triggers problems (closed stream) due to Hadoop handling in e.g. ParquetIO class. + * Stream wrapping solves this. + */ +public class WrappedInputStream extends SeekableInputStream { + + private final SeekableInputStream wrappedInput; + + public WrappedInputStream(SeekableInputStream wrappedInput) { + this.wrappedInput = wrappedInput; + } + + @Override + public long getPos() throws IOException { + return wrappedInput.getPos(); + } + + @Override + public void seek(long newPos) throws IOException { + wrappedInput.seek(newPos); + } + + @Override + public int read() throws IOException { + return wrappedInput.read(); + } + + @Override + public int read(byte[] b, int off, int len) throws IOException { + return wrappedInput.read(b, off, len); + } + + @Override + public void close() throws IOException { + super.close(); + wrappedInput.close(); + } + + @SuppressWarnings("checkstyle:NoFinalizer") + @Override + protected void finalize() throws Throwable { + close(); + } +} From 1ad869bf4efbbc26fa5b14544c6c7617def4be67 Mon Sep 17 00:00:00 2001 From: Gidon Gershinsky Date: Wed, 19 Jan 2022 15:12:25 +0200 Subject: [PATCH 06/11] update the pr --- .../encryption/EncryptedOutputFile.java | 20 ++---- .../iceberg/encryption/EncryptionManager.java | 18 ----- .../NativeFileCryptoParameters.java | 2 +- .../java/org/apache/iceberg/io/InputFile.java | 15 ++++ .../apache/iceberg/aws/s3/S3InputFile.java | 16 ++++- .../org/apache/iceberg/TableProperties.java | 42 ------------ .../encryption/BaseEncryptedOutputFile.java | 12 ++++ .../apache/iceberg/encryption/Ciphers.java | 14 ++-- .../encryption/NativeEncryptedInputFile.java | 63 ----------------- .../encryption/NativeEncryptedOutputFile.java | 56 --------------- .../iceberg/hadoop/HadoopInputFile.java | 12 ++++ .../iceberg/io/FileAppenderFactory.java | 6 ++ .../apache/iceberg/io/WrappedInputStream.java | 68 ------------------- 13 files changed, 78 insertions(+), 266 deletions(-) delete mode 100644 core/src/main/java/org/apache/iceberg/encryption/NativeEncryptedInputFile.java delete mode 100644 core/src/main/java/org/apache/iceberg/encryption/NativeEncryptedOutputFile.java delete mode 100644 core/src/main/java/org/apache/iceberg/io/WrappedInputStream.java diff --git a/api/src/main/java/org/apache/iceberg/encryption/EncryptedOutputFile.java b/api/src/main/java/org/apache/iceberg/encryption/EncryptedOutputFile.java index ddf8d7d0b6f8..e2a24d8fee66 100644 --- a/api/src/main/java/org/apache/iceberg/encryption/EncryptedOutputFile.java +++ b/api/src/main/java/org/apache/iceberg/encryption/EncryptedOutputFile.java @@ -22,36 +22,30 @@ import org.apache.iceberg.io.OutputFile; /** - * TODO update comment - * Thin wrapper around a {@link OutputFile} that is encrypting bytes written to the underlying - * file system, via an encryption key that is symbolized by the enclosed - * {@link EncryptionKeyMetadata}. + * A wrapper around a {@link OutputFile} that is encrypting bytes written to the underlying + * file system. The encryption is performed by either file format (supporting encryption natively, such as + * Parquet or ORC) or by a format-agnostic flat stream encryptor. The enclosed {@link EncryptionKeyMetadata} keeps + * the information required by the authorized readers to retrieve the encryption keys and other crypto parameters, + * necessary to decrypt the file. *

* The {@link EncryptionManager} returns instances of these when passed output files that should * be encrypted as they are being written to the backing file system. */ public interface EncryptedOutputFile { - /** - * Use flat filestream encryption (default) or pushdown to native format encryption - */ - default boolean useNativeEncryption() { - return false; - } - /** * An OutputFile instance that encrypts the bytes that are written to its output streams. */ OutputFile encryptingOutputFile(); /** - * Metadata about the encryption key that is being used to encrypt the associated + * Metadata about the encryption keys and other crypto parameters used to encrypt the associated * {@link #encryptingOutputFile()}. */ EncryptionKeyMetadata keyMetadata(); /** - * Return parameters of native encryption (if the latter is used for this file) + * Parameters of native encryption (if used for this file) */ default NativeFileCryptoParameters nativeEncryptionParameters() { return null; diff --git a/api/src/main/java/org/apache/iceberg/encryption/EncryptionManager.java b/api/src/main/java/org/apache/iceberg/encryption/EncryptionManager.java index 2ba684723e84..97f6f639311b 100644 --- a/api/src/main/java/org/apache/iceberg/encryption/EncryptionManager.java +++ b/api/src/main/java/org/apache/iceberg/encryption/EncryptionManager.java @@ -71,22 +71,4 @@ default Iterable decrypt(Iterable encrypted) { default Iterable encrypt(Iterable rawOutput) { return Iterables.transform(rawOutput, this::encrypt); } - - /** - * Encrypt a manifest list - * @param rawOutput raw output file - * @return encrypted output file - */ - default EncryptedOutputFile encryptManifestList(OutputFile rawOutput) { - return encrypt(rawOutput); - } - - /** - * Encrypt a manifest file - * @param rawOutput raw output file - * @return encrypted output file - */ - default EncryptedOutputFile encryptManifestFile(OutputFile rawOutput) { - return encrypt(rawOutput); - } } diff --git a/api/src/main/java/org/apache/iceberg/encryption/NativeFileCryptoParameters.java b/api/src/main/java/org/apache/iceberg/encryption/NativeFileCryptoParameters.java index 55d16c7c74bc..c771ae24d9c7 100644 --- a/api/src/main/java/org/apache/iceberg/encryption/NativeFileCryptoParameters.java +++ b/api/src/main/java/org/apache/iceberg/encryption/NativeFileCryptoParameters.java @@ -26,7 +26,7 @@ /** * Barebone encryption parameters, one object per content file. - * Carries the file and column encryption keys (and optional AAD prefix). + * Carries the file encryption key (and optional AAD prefix, column keys). */ public class NativeFileCryptoParameters { private ByteBuffer fileAadPrefix; diff --git a/api/src/main/java/org/apache/iceberg/io/InputFile.java b/api/src/main/java/org/apache/iceberg/io/InputFile.java index 6ad3f32a907c..19e6a3483fe0 100644 --- a/api/src/main/java/org/apache/iceberg/io/InputFile.java +++ b/api/src/main/java/org/apache/iceberg/io/InputFile.java @@ -20,6 +20,7 @@ package org.apache.iceberg.io; import java.io.IOException; +import org.apache.iceberg.encryption.NativeFileCryptoParameters; import org.apache.iceberg.exceptions.NotFoundException; import org.apache.iceberg.exceptions.RuntimeIOException; @@ -59,4 +60,18 @@ public interface InputFile { * @return true if the file exists, false otherwise */ boolean exists(); + + // TODO remove this comment after review + // This is a minimum-disruption way to pass native decryption parameters to Parquet and ORC formats. + // Other possible solutions: 1. use a wrapper class (like EncryptedInputStream) with the native decryption parameter - + // this will require changes in dozens of data reading calls/classes. + // 2. add a new class/interface that implements/extends InputStream, and use an instanceof check where required. This + // approach needs to carefully handle the current instanceof checks of HadoopInputStream and HadoopSeekableInputStream + // objects that get class-specific treatment. Otherwise streams won't be closed, etc. + default NativeFileCryptoParameters getNativeDecryptionParameters() { + return null; + } + + default void setNativeDecryptionParameters(NativeFileCryptoParameters nativeDecryptionParameters) { + } } diff --git a/aws/src/main/java/org/apache/iceberg/aws/s3/S3InputFile.java b/aws/src/main/java/org/apache/iceberg/aws/s3/S3InputFile.java index 0ca69aad194d..aa86cd27f0aa 100644 --- a/aws/src/main/java/org/apache/iceberg/aws/s3/S3InputFile.java +++ b/aws/src/main/java/org/apache/iceberg/aws/s3/S3InputFile.java @@ -20,12 +20,16 @@ package org.apache.iceberg.aws.s3; import org.apache.iceberg.aws.AwsProperties; +import org.apache.iceberg.encryption.NativeFileCryptoParameters; +import org.apache.iceberg.encryption.NativelyEncryptedFile; import org.apache.iceberg.io.InputFile; import org.apache.iceberg.io.SeekableInputStream; import org.apache.iceberg.metrics.MetricsContext; import software.amazon.awssdk.services.s3.S3Client; -public class S3InputFile extends BaseS3File implements InputFile { +public class S3InputFile extends BaseS3File implements InputFile, NativelyEncryptedFile { + private NativeFileCryptoParameters nativeDecryptionParameters; + public static S3InputFile fromLocation(String location, S3Client client, AwsProperties awsProperties, MetricsContext metrics) { return new S3InputFile(client, new S3URI(location), awsProperties, metrics); @@ -49,4 +53,14 @@ public long getLength() { public SeekableInputStream newStream() { return new S3InputStream(client(), uri(), awsProperties(), metrics()); } + + @Override + public NativeFileCryptoParameters nativeCryptoParameters() { + return nativeDecryptionParameters; + } + + @Override + public void setNativeCryptoParameters(NativeFileCryptoParameters nativeCryptoParameters) { + this.nativeDecryptionParameters = nativeCryptoParameters; + } } diff --git a/core/src/main/java/org/apache/iceberg/TableProperties.java b/core/src/main/java/org/apache/iceberg/TableProperties.java index 43c3cc72ed77..b9b54e76cb5d 100644 --- a/core/src/main/java/org/apache/iceberg/TableProperties.java +++ b/core/src/main/java/org/apache/iceberg/TableProperties.java @@ -276,50 +276,8 @@ private TableProperties() { @Deprecated public static final boolean MERGE_CARDINALITY_CHECK_ENABLED_DEFAULT = true; -<<<<<<< HEAD public static final String MERGE_DISTRIBUTION_MODE = "write.merge.distribution-mode"; public static final String UPSERT_ENABLED = "write.upsert.enabled"; public static final boolean UPSERT_ENABLED_DEFAULT = false; -======= - /** - * Encryption manager type - */ - public static final String ENCRYPTION_MANAGER_TYPE = "encryption.manager.type"; - public static final String ENCRYPTION_MANAGER_TYPE_PLAINTEXT = "plaintext"; - public static final String ENCRYPTION_MANAGER_TYPE_LEGACY = "legacy"; // TODO needed? - public static final String ENCRYPTION_MANAGER_TYPE_SINGLE_ENVELOPE = "envelope"; - public static final String ENCRYPTION_MANAGER_TYPE_DOUBLE_ENVELOPE = "double.envelope"; - - public static final String ENCRYPTION_TABLE_KEY = "encryption.table.key"; - - public static final String ENCRYPTION_DEK_LENGTH = "encryption.data.key.length"; - public static final int ENCRYPTION_DEK_LENGTH_DEFAULT = 16; - - public static final String ENCRYPTION_DATA_ALGORITHM = "encryption.data.algorithm"; - public static final String ENCRYPTION_DATA_ALGORITHM_DEFAULT = EncryptionAlgorithm.AES_GCM.toString(); - - /** - * Leverage file format native encryption instead of encrypting the entire file through Iceberg encryption stream - */ - public static final String ENCRYPTION_PUSHDOWN_ENABLED = "encryption.pushdown"; - public static final boolean ENCRYPTION_PUSHDOWN_ENABLED_DEFAULT = true; - - /** - * Implementation of the KMS client for envelope encryption. - */ - public static final String ENCRYPTION_KMS_CLIENT_IMPL = "encryption.kms.client-impl"; - - /** - * Implementation of custom out/input files (for metadata and Avro data) in envelope encryption. - */ - public static final String ENCRYPTION_OUTPUT_FILE_IMPL = "encryption.output.file-impl"; - public static final String ENCRYPTION_INPUT_FILE_IMPL = "encryption.input.file-impl"; - - /** - * Implementation of legacy encryption manager. - * TODO needed? - */ - public static final String ENCRYPTION_LEGACY_MANAGER_IMPL = "encryption.legacy.manager-impl"; ->>>>>>> d4aad3190 (updates after review round 1) } diff --git a/core/src/main/java/org/apache/iceberg/encryption/BaseEncryptedOutputFile.java b/core/src/main/java/org/apache/iceberg/encryption/BaseEncryptedOutputFile.java index ab758b2c7171..cc7875d216b3 100644 --- a/core/src/main/java/org/apache/iceberg/encryption/BaseEncryptedOutputFile.java +++ b/core/src/main/java/org/apache/iceberg/encryption/BaseEncryptedOutputFile.java @@ -25,10 +25,17 @@ class BaseEncryptedOutputFile implements EncryptedOutputFile { private final OutputFile encryptingOutputFile; private final EncryptionKeyMetadata keyMetadata; + private final NativeFileCryptoParameters nativeEncryptionParameters; BaseEncryptedOutputFile(OutputFile encryptingOutputFile, EncryptionKeyMetadata keyMetadata) { + this(encryptingOutputFile, keyMetadata, null); + } + + BaseEncryptedOutputFile(OutputFile encryptingOutputFile, EncryptionKeyMetadata keyMetadata, + NativeFileCryptoParameters nativeEncryptionParameters) { this.encryptingOutputFile = encryptingOutputFile; this.keyMetadata = keyMetadata; + this.nativeEncryptionParameters = nativeEncryptionParameters; } @Override @@ -40,4 +47,9 @@ public OutputFile encryptingOutputFile() { public EncryptionKeyMetadata keyMetadata() { return keyMetadata; } + + @Override + public NativeFileCryptoParameters nativeEncryptionParameters() { + return nativeEncryptionParameters; + } } diff --git a/core/src/main/java/org/apache/iceberg/encryption/Ciphers.java b/core/src/main/java/org/apache/iceberg/encryption/Ciphers.java index 4a2cc772bd4c..65023a3e1da8 100644 --- a/core/src/main/java/org/apache/iceberg/encryption/Ciphers.java +++ b/core/src/main/java/org/apache/iceberg/encryption/Ciphers.java @@ -37,7 +37,7 @@ public static class AesGcmEncryptor { private final Cipher cipher; private final SecureRandom randomGenerator; - AesGcmEncryptor(byte[] keyBytes) { + public AesGcmEncryptor(byte[] keyBytes) { int keyLength = keyBytes.length; if (!(keyLength == 16 || keyLength == 24 || keyLength == 32)) { throw new IllegalArgumentException("Wrong key length " + keyLength); @@ -53,7 +53,7 @@ public static class AesGcmEncryptor { this.randomGenerator = new SecureRandom(); } - public byte[] encrypt(byte[] plainText) { + public byte[] encrypt(byte[] plainText, byte[] aad) { byte[] nonce = new byte[NONCE_LENGTH]; randomGenerator.nextBytes(nonce); int cipherTextLength = NONCE_LENGTH + plainText.length + GCM_TAG_LENGTH; @@ -62,6 +62,9 @@ public byte[] encrypt(byte[] plainText) { try { GCMParameterSpec spec = new GCMParameterSpec(GCM_TAG_LENGTH_BITS, nonce); cipher.init(Cipher.ENCRYPT_MODE, aesKey, spec); + if (null != aad) { + cipher.updateAAD(aad); + } cipher.doFinal(plainText, 0, plainText.length, cipherText, NONCE_LENGTH); } catch (GeneralSecurityException e) { throw new RuntimeException("Failed to encrypt", e); @@ -78,7 +81,7 @@ public static class AesGcmDecryptor { private final SecretKeySpec aesKey; private final Cipher cipher; - AesGcmDecryptor(byte[] keyBytes) { + public AesGcmDecryptor(byte[] keyBytes) { int keyLength = keyBytes.length; if (!(keyLength == 16 || keyLength == 24 || keyLength == 32)) { throw new IllegalArgumentException("Wrong key length " + keyLength); @@ -93,7 +96,7 @@ public static class AesGcmDecryptor { } } - public byte[] decrypt(byte[] ciphertext) { + public byte[] decrypt(byte[] ciphertext, byte[] aad) { int plainTextLength = ciphertext.length - AesGcmEncryptor.GCM_TAG_LENGTH - AesGcmEncryptor.NONCE_LENGTH; if (plainTextLength < 1) { throw new RuntimeException("Wrong input length " + plainTextLength); @@ -108,6 +111,9 @@ public byte[] decrypt(byte[] ciphertext) { try { GCMParameterSpec spec = new GCMParameterSpec(AesGcmEncryptor.GCM_TAG_LENGTH_BITS, nonce); cipher.init(Cipher.DECRYPT_MODE, aesKey, spec); + if (null != aad) { + cipher.updateAAD(aad); + } cipher.doFinal(ciphertext, AesGcmEncryptor.NONCE_LENGTH, inputLength, plainText, 0); } catch (AEADBadTagException e) { throw new RuntimeException("GCM tag check failed", e); diff --git a/core/src/main/java/org/apache/iceberg/encryption/NativeEncryptedInputFile.java b/core/src/main/java/org/apache/iceberg/encryption/NativeEncryptedInputFile.java deleted file mode 100644 index 178c35db29c6..000000000000 --- a/core/src/main/java/org/apache/iceberg/encryption/NativeEncryptedInputFile.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iceberg.encryption; - -import org.apache.iceberg.io.InputFile; -import org.apache.iceberg.io.SeekableInputStream; -import org.apache.iceberg.io.WrappedInputStream; - -public class NativeEncryptedInputFile implements InputFile { - - private final InputFile rawInput; - private final NativeFileCryptoParameters nativeDecryptionParameteres; - - NativeEncryptedInputFile(InputFile rawInput, NativeFileCryptoParameters nativeDecryptionParameteres) { - this.rawInput = rawInput; - this.nativeDecryptionParameteres = nativeDecryptionParameteres; - } - - public NativeFileCryptoParameters nativeDecryptionParameters() { - return nativeDecryptionParameteres; - } - - @Override - public long getLength() { - return rawInput.getLength(); - } - - @Override - public SeekableInputStream newStream() { - // TODO remove this comment after review - // This class is not HadoopInputFile, while its rawInput can be; and rawInput's stream can be FSDataInputStream. - // Returning rawInput.newStream() here leads to closed stream exceptions, due to Hadoop handling in ParquetIO class. - // Therefore, using stream wrap. - return new WrappedInputStream(rawInput.newStream()); - } - - @Override - public String location() { - return rawInput.location(); - } - - @Override - public boolean exists() { - return rawInput.exists(); - } -} diff --git a/core/src/main/java/org/apache/iceberg/encryption/NativeEncryptedOutputFile.java b/core/src/main/java/org/apache/iceberg/encryption/NativeEncryptedOutputFile.java deleted file mode 100644 index 7ddd489cefbf..000000000000 --- a/core/src/main/java/org/apache/iceberg/encryption/NativeEncryptedOutputFile.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iceberg.encryption; - -import org.apache.iceberg.io.OutputFile; - -class NativeEncryptedOutputFile implements EncryptedOutputFile { - - private final OutputFile rawOutput; - private final NativeFileCryptoParameters nativeEncryptionParameters; - private final EncryptionKeyMetadata keyMetadata; - - NativeEncryptedOutputFile(OutputFile rawOutput, EncryptionKeyMetadata keyMetadata, - NativeFileCryptoParameters nativeEncryptionParameters) { - this.rawOutput = rawOutput; - this.nativeEncryptionParameters = nativeEncryptionParameters; - this.keyMetadata = keyMetadata; - } - - @Override - public OutputFile encryptingOutputFile() { - return rawOutput; - } - - @Override - public EncryptionKeyMetadata keyMetadata() { - return keyMetadata; - } - - @Override - public NativeFileCryptoParameters nativeEncryptionParameters() { - return nativeEncryptionParameters; - } - - @Override - public boolean useNativeEncryption() { - return true; - } -} diff --git a/core/src/main/java/org/apache/iceberg/hadoop/HadoopInputFile.java b/core/src/main/java/org/apache/iceberg/hadoop/HadoopInputFile.java index 7cf6dee60b82..66fe88fd7161 100644 --- a/core/src/main/java/org/apache/iceberg/hadoop/HadoopInputFile.java +++ b/core/src/main/java/org/apache/iceberg/hadoop/HadoopInputFile.java @@ -28,6 +28,7 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.iceberg.encryption.NativeFileCryptoParameters; import org.apache.iceberg.exceptions.NotFoundException; import org.apache.iceberg.exceptions.RuntimeIOException; import org.apache.iceberg.io.InputFile; @@ -49,6 +50,7 @@ public class HadoopInputFile implements InputFile { private final Configuration conf; private FileStatus stat = null; private Long length = null; + private NativeFileCryptoParameters nativeDecryptionParameters; public static HadoopInputFile fromLocation(CharSequence location, Configuration conf) { FileSystem fs = Util.getFs(new Path(location.toString()), conf); @@ -224,6 +226,16 @@ public boolean exists() { } } + @Override + public NativeFileCryptoParameters getNativeDecryptionParameters() { + return nativeDecryptionParameters; + } + + @Override + public void setNativeDecryptionParameters(NativeFileCryptoParameters nativeDecryptionParameters) { + this.nativeDecryptionParameters = nativeDecryptionParameters; + } + @Override public String toString() { return path.toString(); diff --git a/core/src/main/java/org/apache/iceberg/io/FileAppenderFactory.java b/core/src/main/java/org/apache/iceberg/io/FileAppenderFactory.java index b093eab447fe..b0e163dfb82e 100644 --- a/core/src/main/java/org/apache/iceberg/io/FileAppenderFactory.java +++ b/core/src/main/java/org/apache/iceberg/io/FileAppenderFactory.java @@ -32,6 +32,7 @@ */ public interface FileAppenderFactory { + // TODO deprecate/remove. Used in tests only. /** * Create a new {@link FileAppender}. * @@ -41,6 +42,11 @@ public interface FileAppenderFactory { */ FileAppender newAppender(OutputFile outputFile, FileFormat fileFormat); + // TODO document this, or change the previous function + default FileAppender newAppender(EncryptedOutputFile outputFile, FileFormat fileFormat) { + return null; + } + /** * Create a new {@link DataWriter}. * diff --git a/core/src/main/java/org/apache/iceberg/io/WrappedInputStream.java b/core/src/main/java/org/apache/iceberg/io/WrappedInputStream.java deleted file mode 100644 index 42fe90a1ac6e..000000000000 --- a/core/src/main/java/org/apache/iceberg/io/WrappedInputStream.java +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iceberg.io; - -import java.io.IOException; - -/** - * Wrapping InputFiles can have HadoopInputFile as rawInput. Therefore, rawInput's stream can be - * FSDataInputStream. This triggers problems (closed stream) due to Hadoop handling in e.g. ParquetIO class. - * Stream wrapping solves this. - */ -public class WrappedInputStream extends SeekableInputStream { - - private final SeekableInputStream wrappedInput; - - public WrappedInputStream(SeekableInputStream wrappedInput) { - this.wrappedInput = wrappedInput; - } - - @Override - public long getPos() throws IOException { - return wrappedInput.getPos(); - } - - @Override - public void seek(long newPos) throws IOException { - wrappedInput.seek(newPos); - } - - @Override - public int read() throws IOException { - return wrappedInput.read(); - } - - @Override - public int read(byte[] b, int off, int len) throws IOException { - return wrappedInput.read(b, off, len); - } - - @Override - public void close() throws IOException { - super.close(); - wrappedInput.close(); - } - - @SuppressWarnings("checkstyle:NoFinalizer") - @Override - protected void finalize() throws Throwable { - close(); - } -} From 3db6f8958492343828da5b9152d2a865c4e344d6 Mon Sep 17 00:00:00 2001 From: Gidon Gershinsky Date: Tue, 25 Jan 2022 14:37:41 +0200 Subject: [PATCH 07/11] post-review changes --- .../encryption/EncryptedOutputFile.java | 17 ++----- .../java/org/apache/iceberg/io/InputFile.java | 15 ------ .../encryption/BaseEncryptedOutputFile.java | 12 ----- .../apache/iceberg/encryption/Ciphers.java | 22 ++++----- .../NativeFileCryptoParameters.java | 46 ++++--------------- .../encryption/NativelyEncryptedFile.java | 29 ++++++++++++ .../iceberg/hadoop/HadoopInputFile.java | 9 ++-- .../iceberg/io/FileAppenderFactory.java | 6 --- 8 files changed, 58 insertions(+), 98 deletions(-) rename {api => core}/src/main/java/org/apache/iceberg/encryption/NativeFileCryptoParameters.java (57%) create mode 100644 core/src/main/java/org/apache/iceberg/encryption/NativelyEncryptedFile.java diff --git a/api/src/main/java/org/apache/iceberg/encryption/EncryptedOutputFile.java b/api/src/main/java/org/apache/iceberg/encryption/EncryptedOutputFile.java index e2a24d8fee66..d05033ebe150 100644 --- a/api/src/main/java/org/apache/iceberg/encryption/EncryptedOutputFile.java +++ b/api/src/main/java/org/apache/iceberg/encryption/EncryptedOutputFile.java @@ -22,11 +22,9 @@ import org.apache.iceberg.io.OutputFile; /** - * A wrapper around a {@link OutputFile} that is encrypting bytes written to the underlying - * file system. The encryption is performed by either file format (supporting encryption natively, such as - * Parquet or ORC) or by a format-agnostic flat stream encryptor. The enclosed {@link EncryptionKeyMetadata} keeps - * the information required by the authorized readers to retrieve the encryption keys and other crypto parameters, - * necessary to decrypt the file. + * Thin wrapper around a {@link OutputFile} that is encrypting bytes written to the underlying + * file system, via an encryption key that is symbolized by the enclosed + * {@link EncryptionKeyMetadata}. *

* The {@link EncryptionManager} returns instances of these when passed output files that should * be encrypted as they are being written to the backing file system. @@ -39,15 +37,8 @@ public interface EncryptedOutputFile { OutputFile encryptingOutputFile(); /** - * Metadata about the encryption keys and other crypto parameters used to encrypt the associated + * Metadata about the encryption key that is being used to encrypt the associated * {@link #encryptingOutputFile()}. */ EncryptionKeyMetadata keyMetadata(); - - /** - * Parameters of native encryption (if used for this file) - */ - default NativeFileCryptoParameters nativeEncryptionParameters() { - return null; - } } diff --git a/api/src/main/java/org/apache/iceberg/io/InputFile.java b/api/src/main/java/org/apache/iceberg/io/InputFile.java index 19e6a3483fe0..6ad3f32a907c 100644 --- a/api/src/main/java/org/apache/iceberg/io/InputFile.java +++ b/api/src/main/java/org/apache/iceberg/io/InputFile.java @@ -20,7 +20,6 @@ package org.apache.iceberg.io; import java.io.IOException; -import org.apache.iceberg.encryption.NativeFileCryptoParameters; import org.apache.iceberg.exceptions.NotFoundException; import org.apache.iceberg.exceptions.RuntimeIOException; @@ -60,18 +59,4 @@ public interface InputFile { * @return true if the file exists, false otherwise */ boolean exists(); - - // TODO remove this comment after review - // This is a minimum-disruption way to pass native decryption parameters to Parquet and ORC formats. - // Other possible solutions: 1. use a wrapper class (like EncryptedInputStream) with the native decryption parameter - - // this will require changes in dozens of data reading calls/classes. - // 2. add a new class/interface that implements/extends InputStream, and use an instanceof check where required. This - // approach needs to carefully handle the current instanceof checks of HadoopInputStream and HadoopSeekableInputStream - // objects that get class-specific treatment. Otherwise streams won't be closed, etc. - default NativeFileCryptoParameters getNativeDecryptionParameters() { - return null; - } - - default void setNativeDecryptionParameters(NativeFileCryptoParameters nativeDecryptionParameters) { - } } diff --git a/core/src/main/java/org/apache/iceberg/encryption/BaseEncryptedOutputFile.java b/core/src/main/java/org/apache/iceberg/encryption/BaseEncryptedOutputFile.java index cc7875d216b3..ab758b2c7171 100644 --- a/core/src/main/java/org/apache/iceberg/encryption/BaseEncryptedOutputFile.java +++ b/core/src/main/java/org/apache/iceberg/encryption/BaseEncryptedOutputFile.java @@ -25,17 +25,10 @@ class BaseEncryptedOutputFile implements EncryptedOutputFile { private final OutputFile encryptingOutputFile; private final EncryptionKeyMetadata keyMetadata; - private final NativeFileCryptoParameters nativeEncryptionParameters; BaseEncryptedOutputFile(OutputFile encryptingOutputFile, EncryptionKeyMetadata keyMetadata) { - this(encryptingOutputFile, keyMetadata, null); - } - - BaseEncryptedOutputFile(OutputFile encryptingOutputFile, EncryptionKeyMetadata keyMetadata, - NativeFileCryptoParameters nativeEncryptionParameters) { this.encryptingOutputFile = encryptingOutputFile; this.keyMetadata = keyMetadata; - this.nativeEncryptionParameters = nativeEncryptionParameters; } @Override @@ -47,9 +40,4 @@ public OutputFile encryptingOutputFile() { public EncryptionKeyMetadata keyMetadata() { return keyMetadata; } - - @Override - public NativeFileCryptoParameters nativeEncryptionParameters() { - return nativeEncryptionParameters; - } } diff --git a/core/src/main/java/org/apache/iceberg/encryption/Ciphers.java b/core/src/main/java/org/apache/iceberg/encryption/Ciphers.java index 65023a3e1da8..5f57bb19acb3 100644 --- a/core/src/main/java/org/apache/iceberg/encryption/Ciphers.java +++ b/core/src/main/java/org/apache/iceberg/encryption/Ciphers.java @@ -27,12 +27,11 @@ import javax.crypto.spec.SecretKeySpec; public class Ciphers { + private static final int NONCE_LENGTH = 12; + private static final int GCM_TAG_LENGTH = 16; + private static final int GCM_TAG_LENGTH_BITS = 8 * GCM_TAG_LENGTH; public static class AesGcmEncryptor { - public static final int NONCE_LENGTH = 12; - public static final int GCM_TAG_LENGTH = 16; - public static final int GCM_TAG_LENGTH_BITS = 8 * GCM_TAG_LENGTH; - private final SecretKeySpec aesKey; private final Cipher cipher; private final SecureRandom randomGenerator; @@ -40,7 +39,8 @@ public static class AesGcmEncryptor { public AesGcmEncryptor(byte[] keyBytes) { int keyLength = keyBytes.length; if (!(keyLength == 16 || keyLength == 24 || keyLength == 32)) { - throw new IllegalArgumentException("Wrong key length " + keyLength); + throw new IllegalArgumentException("Wrong key length " + keyLength + + ". Must be 16, 24 or 32 bytes"); } this.aesKey = new SecretKeySpec(keyBytes, "AES"); @@ -97,24 +97,24 @@ public AesGcmDecryptor(byte[] keyBytes) { } public byte[] decrypt(byte[] ciphertext, byte[] aad) { - int plainTextLength = ciphertext.length - AesGcmEncryptor.GCM_TAG_LENGTH - AesGcmEncryptor.NONCE_LENGTH; + int plainTextLength = ciphertext.length - GCM_TAG_LENGTH - NONCE_LENGTH; if (plainTextLength < 1) { throw new RuntimeException("Wrong input length " + plainTextLength); } // Get the nonce from ciphertext - byte[] nonce = new byte[AesGcmEncryptor.NONCE_LENGTH]; - System.arraycopy(ciphertext, 0, nonce, 0, AesGcmEncryptor.NONCE_LENGTH); + byte[] nonce = new byte[NONCE_LENGTH]; + System.arraycopy(ciphertext, 0, nonce, 0, NONCE_LENGTH); byte[] plainText = new byte[plainTextLength]; - int inputLength = ciphertext.length - AesGcmEncryptor.NONCE_LENGTH; + int inputLength = ciphertext.length - NONCE_LENGTH; try { - GCMParameterSpec spec = new GCMParameterSpec(AesGcmEncryptor.GCM_TAG_LENGTH_BITS, nonce); + GCMParameterSpec spec = new GCMParameterSpec(GCM_TAG_LENGTH_BITS, nonce); cipher.init(Cipher.DECRYPT_MODE, aesKey, spec); if (null != aad) { cipher.updateAAD(aad); } - cipher.doFinal(ciphertext, AesGcmEncryptor.NONCE_LENGTH, inputLength, plainText, 0); + cipher.doFinal(ciphertext, NONCE_LENGTH, inputLength, plainText, 0); } catch (AEADBadTagException e) { throw new RuntimeException("GCM tag check failed", e); } catch (GeneralSecurityException e) { diff --git a/api/src/main/java/org/apache/iceberg/encryption/NativeFileCryptoParameters.java b/core/src/main/java/org/apache/iceberg/encryption/NativeFileCryptoParameters.java similarity index 57% rename from api/src/main/java/org/apache/iceberg/encryption/NativeFileCryptoParameters.java rename to core/src/main/java/org/apache/iceberg/encryption/NativeFileCryptoParameters.java index c771ae24d9c7..4b04b8bd6749 100644 --- a/api/src/main/java/org/apache/iceberg/encryption/NativeFileCryptoParameters.java +++ b/core/src/main/java/org/apache/iceberg/encryption/NativeFileCryptoParameters.java @@ -17,35 +17,29 @@ * under the License. */ - package org.apache.iceberg.encryption; import java.nio.ByteBuffer; -import java.util.Map; import org.apache.iceberg.relocated.com.google.common.base.Preconditions; /** * Barebone encryption parameters, one object per content file. - * Carries the file encryption key (and optional AAD prefix, column keys). + * Carries the file encryption key (later, will be extended with column keys and AAD prefix). + * Applicable only to formats with native encryption support (Parquet and ORC). */ public class NativeFileCryptoParameters { - private ByteBuffer fileAadPrefix; - private Map columnKeys; private ByteBuffer fileKey; private String fileEncryptionAlgorithm; - private NativeFileCryptoParameters(Map columnKeys, ByteBuffer fileKey, - ByteBuffer fileAadPrefix, String fileEncryptionAlgorithm) { - Preconditions.checkState((columnKeys != null && columnKeys.size() > 0) || fileKey != null, - "No file or column keys are supplied"); - this.columnKeys = columnKeys; + private NativeFileCryptoParameters(ByteBuffer fileKey, String fileEncryptionAlgorithm) { + Preconditions.checkState(fileKey != null, "File encryption key is not supplied"); this.fileKey = fileKey; - this.fileAadPrefix = fileAadPrefix; this.fileEncryptionAlgorithm = fileEncryptionAlgorithm; } /** * Creates the builder. + * * @param fileKey per-file encryption key. For example, used as "footer key" DEK in Parquet encryption. */ public static Builder create(ByteBuffer fileKey) { @@ -53,8 +47,6 @@ public static Builder create(ByteBuffer fileKey) { } public static class Builder { - private ByteBuffer fileAadPrefix; - private Map columnKeys; private ByteBuffer fileKey; private String fileEncryptionAlgorithm; @@ -62,44 +54,24 @@ private Builder(ByteBuffer fileKey) { this.fileKey = fileKey; } - /** - * Set column encryption keys. - * @param columnKeyMap Map of column names to column keys. Column names must be the original names, - * used during content file creation. For example, Parquet will use them to find and - * encrypt the relevant columns. - */ - public Builder columnKeys(Map columnKeyMap) { - this.columnKeys = columnKeyMap; - return this; - } - - public Builder aadPrefix(ByteBuffer aadPrefix) { - this.fileAadPrefix = aadPrefix; - return this; - } - public Builder encryptionAlgorithm(String encryptionAlgorithm) { this.fileEncryptionAlgorithm = encryptionAlgorithm; return this; } public NativeFileCryptoParameters build() { - return new NativeFileCryptoParameters(columnKeys, fileKey, fileAadPrefix, fileEncryptionAlgorithm); + return new NativeFileCryptoParameters(fileKey, fileEncryptionAlgorithm); } - } - public ByteBuffer aadPrefix() { - return fileAadPrefix; + // TODO add back column encryption keys + + // TODO add back AAD prefix (cryptographic file identity) } public ByteBuffer fileKey() { return fileKey; } - public Map columnKeys() { - return columnKeys; - } - public String encryptionAlgorithm() { return fileEncryptionAlgorithm; } diff --git a/core/src/main/java/org/apache/iceberg/encryption/NativelyEncryptedFile.java b/core/src/main/java/org/apache/iceberg/encryption/NativelyEncryptedFile.java new file mode 100644 index 000000000000..4f8a2f39e352 --- /dev/null +++ b/core/src/main/java/org/apache/iceberg/encryption/NativelyEncryptedFile.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iceberg.encryption; + +/** + * a minimum client interface to connect to a key management service (KMS). + */ +public interface NativelyEncryptedFile { + NativeFileCryptoParameters getNativeCryptoParameters(); + + void setNativeCryptoParameters(NativeFileCryptoParameters nativeDecryptionParameters); +} diff --git a/core/src/main/java/org/apache/iceberg/hadoop/HadoopInputFile.java b/core/src/main/java/org/apache/iceberg/hadoop/HadoopInputFile.java index 66fe88fd7161..b086ab86875c 100644 --- a/core/src/main/java/org/apache/iceberg/hadoop/HadoopInputFile.java +++ b/core/src/main/java/org/apache/iceberg/hadoop/HadoopInputFile.java @@ -29,6 +29,7 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.iceberg.encryption.NativeFileCryptoParameters; +import org.apache.iceberg.encryption.NativelyEncryptedFile; import org.apache.iceberg.exceptions.NotFoundException; import org.apache.iceberg.exceptions.RuntimeIOException; import org.apache.iceberg.io.InputFile; @@ -41,7 +42,7 @@ *

* This class is based on Parquet's HadoopInputFile. */ -public class HadoopInputFile implements InputFile { +public class HadoopInputFile implements InputFile, NativelyEncryptedFile { public static final String[] NO_LOCATION_PREFERENCE = new String[0]; private final String location; @@ -227,13 +228,13 @@ public boolean exists() { } @Override - public NativeFileCryptoParameters getNativeDecryptionParameters() { + public NativeFileCryptoParameters getNativeCryptoParameters() { return nativeDecryptionParameters; } @Override - public void setNativeDecryptionParameters(NativeFileCryptoParameters nativeDecryptionParameters) { - this.nativeDecryptionParameters = nativeDecryptionParameters; + public void setNativeCryptoParameters(NativeFileCryptoParameters nativeCryptoParameters) { + this.nativeDecryptionParameters = nativeCryptoParameters; } @Override diff --git a/core/src/main/java/org/apache/iceberg/io/FileAppenderFactory.java b/core/src/main/java/org/apache/iceberg/io/FileAppenderFactory.java index b0e163dfb82e..b093eab447fe 100644 --- a/core/src/main/java/org/apache/iceberg/io/FileAppenderFactory.java +++ b/core/src/main/java/org/apache/iceberg/io/FileAppenderFactory.java @@ -32,7 +32,6 @@ */ public interface FileAppenderFactory { - // TODO deprecate/remove. Used in tests only. /** * Create a new {@link FileAppender}. * @@ -42,11 +41,6 @@ public interface FileAppenderFactory { */ FileAppender newAppender(OutputFile outputFile, FileFormat fileFormat); - // TODO document this, or change the previous function - default FileAppender newAppender(EncryptedOutputFile outputFile, FileFormat fileFormat) { - return null; - } - /** * Create a new {@link DataWriter}. * From 12e38de10c5e6804d2fc9b1d42576644d3bd178d Mon Sep 17 00:00:00 2001 From: Gidon Gershinsky Date: Thu, 27 Jan 2022 14:03:54 +0200 Subject: [PATCH 08/11] post-review changes 2 --- .../apache/iceberg/aws/s3/S3OutputFile.java | 16 +++++- .../apache/iceberg/encryption/Ciphers.java | 7 +-- .../encryption/NativelyEncryptedFile.java | 8 +-- .../iceberg/hadoop/HadoopInputFile.java | 2 +- .../iceberg/hadoop/HadoopOutputFile.java | 15 +++++- .../iceberg/encryption/TestCiphers.java | 53 +++++++++++++++++++ 6 files changed, 92 insertions(+), 9 deletions(-) create mode 100644 core/src/test/java/org/apache/iceberg/encryption/TestCiphers.java diff --git a/aws/src/main/java/org/apache/iceberg/aws/s3/S3OutputFile.java b/aws/src/main/java/org/apache/iceberg/aws/s3/S3OutputFile.java index 48b2ee4f0cae..a8bb3b927bd8 100644 --- a/aws/src/main/java/org/apache/iceberg/aws/s3/S3OutputFile.java +++ b/aws/src/main/java/org/apache/iceberg/aws/s3/S3OutputFile.java @@ -22,6 +22,8 @@ import java.io.IOException; import java.io.UncheckedIOException; import org.apache.iceberg.aws.AwsProperties; +import org.apache.iceberg.encryption.NativeFileCryptoParameters; +import org.apache.iceberg.encryption.NativelyEncryptedFile; import org.apache.iceberg.exceptions.AlreadyExistsException; import org.apache.iceberg.io.InputFile; import org.apache.iceberg.io.OutputFile; @@ -29,7 +31,9 @@ import org.apache.iceberg.metrics.MetricsContext; import software.amazon.awssdk.services.s3.S3Client; -public class S3OutputFile extends BaseS3File implements OutputFile { +public class S3OutputFile extends BaseS3File implements OutputFile, NativelyEncryptedFile { + private NativeFileCryptoParameters nativeEncryptionParameters; + public static S3OutputFile fromLocation(String location, S3Client client, AwsProperties awsProperties, MetricsContext metrics) { return new S3OutputFile(client, new S3URI(location), awsProperties, metrics); @@ -67,4 +71,14 @@ public PositionOutputStream createOrOverwrite() { public InputFile toInputFile() { return new S3InputFile(client(), uri(), awsProperties(), metrics()); } + + @Override + public NativeFileCryptoParameters nativeCryptoParameters() { + return nativeEncryptionParameters; + } + + @Override + public void setNativeCryptoParameters(NativeFileCryptoParameters nativeCryptoParameters) { + this.nativeEncryptionParameters = nativeCryptoParameters; + } } diff --git a/core/src/main/java/org/apache/iceberg/encryption/Ciphers.java b/core/src/main/java/org/apache/iceberg/encryption/Ciphers.java index 5f57bb19acb3..be26bb35ee57 100644 --- a/core/src/main/java/org/apache/iceberg/encryption/Ciphers.java +++ b/core/src/main/java/org/apache/iceberg/encryption/Ciphers.java @@ -39,8 +39,8 @@ public static class AesGcmEncryptor { public AesGcmEncryptor(byte[] keyBytes) { int keyLength = keyBytes.length; if (!(keyLength == 16 || keyLength == 24 || keyLength == 32)) { - throw new IllegalArgumentException("Wrong key length " + keyLength + - ". Must be 16, 24 or 32 bytes"); + throw new IllegalArgumentException("Cannot use a key of length " + keyLength + + " because AES only allows 16, 24 or 32 bytes"); } this.aesKey = new SecretKeySpec(keyBytes, "AES"); @@ -84,7 +84,8 @@ public static class AesGcmDecryptor { public AesGcmDecryptor(byte[] keyBytes) { int keyLength = keyBytes.length; if (!(keyLength == 16 || keyLength == 24 || keyLength == 32)) { - throw new IllegalArgumentException("Wrong key length " + keyLength); + throw new IllegalArgumentException("Cannot use a key of length " + keyLength + + " because AES only allows 16, 24 or 32 bytes"); } this.aesKey = new SecretKeySpec(keyBytes, "AES"); diff --git a/core/src/main/java/org/apache/iceberg/encryption/NativelyEncryptedFile.java b/core/src/main/java/org/apache/iceberg/encryption/NativelyEncryptedFile.java index 4f8a2f39e352..2e0f403397dc 100644 --- a/core/src/main/java/org/apache/iceberg/encryption/NativelyEncryptedFile.java +++ b/core/src/main/java/org/apache/iceberg/encryption/NativelyEncryptedFile.java @@ -20,10 +20,12 @@ package org.apache.iceberg.encryption; /** - * a minimum client interface to connect to a key management service (KMS). + * This interface is applied to OutputFile and InputFile implementations, in order to enable delivery of crypto + * parameters (such as encryption keys etc) from the Iceberg key management module to the writers/readers of file + * formats that support encryption natively (Parquet and ORC). */ public interface NativelyEncryptedFile { - NativeFileCryptoParameters getNativeCryptoParameters(); + NativeFileCryptoParameters nativeCryptoParameters(); - void setNativeCryptoParameters(NativeFileCryptoParameters nativeDecryptionParameters); + void setNativeCryptoParameters(NativeFileCryptoParameters nativeCryptoParameters); } diff --git a/core/src/main/java/org/apache/iceberg/hadoop/HadoopInputFile.java b/core/src/main/java/org/apache/iceberg/hadoop/HadoopInputFile.java index b086ab86875c..7393c91ce32b 100644 --- a/core/src/main/java/org/apache/iceberg/hadoop/HadoopInputFile.java +++ b/core/src/main/java/org/apache/iceberg/hadoop/HadoopInputFile.java @@ -228,7 +228,7 @@ public boolean exists() { } @Override - public NativeFileCryptoParameters getNativeCryptoParameters() { + public NativeFileCryptoParameters nativeCryptoParameters() { return nativeDecryptionParameters; } diff --git a/core/src/main/java/org/apache/iceberg/hadoop/HadoopOutputFile.java b/core/src/main/java/org/apache/iceberg/hadoop/HadoopOutputFile.java index 921a7628e8f5..764725de5d0c 100644 --- a/core/src/main/java/org/apache/iceberg/hadoop/HadoopOutputFile.java +++ b/core/src/main/java/org/apache/iceberg/hadoop/HadoopOutputFile.java @@ -24,6 +24,8 @@ import org.apache.hadoop.fs.FileAlreadyExistsException; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.iceberg.encryption.NativeFileCryptoParameters; +import org.apache.iceberg.encryption.NativelyEncryptedFile; import org.apache.iceberg.exceptions.AlreadyExistsException; import org.apache.iceberg.exceptions.RuntimeIOException; import org.apache.iceberg.io.InputFile; @@ -33,11 +35,12 @@ /** * {@link OutputFile} implementation using the Hadoop {@link FileSystem} API. */ -public class HadoopOutputFile implements OutputFile { +public class HadoopOutputFile implements OutputFile, NativelyEncryptedFile { private final FileSystem fs; private final Path path; private final Configuration conf; + private NativeFileCryptoParameters nativeEncryptionParameters; public static OutputFile fromLocation(CharSequence location, Configuration conf) { Path path = new Path(location.toString()); @@ -114,4 +117,14 @@ public InputFile toInputFile() { public String toString() { return location(); } + + @Override + public NativeFileCryptoParameters nativeCryptoParameters() { + return nativeEncryptionParameters; + } + + @Override + public void setNativeCryptoParameters(NativeFileCryptoParameters nativeCryptoParameters) { + this.nativeEncryptionParameters = nativeCryptoParameters; + } } diff --git a/core/src/test/java/org/apache/iceberg/encryption/TestCiphers.java b/core/src/test/java/org/apache/iceberg/encryption/TestCiphers.java new file mode 100644 index 000000000000..8992072ec656 --- /dev/null +++ b/core/src/test/java/org/apache/iceberg/encryption/TestCiphers.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iceberg.encryption; + +import java.nio.charset.StandardCharsets; +import java.security.SecureRandom; +import org.junit.Assert; +import org.junit.Test; + +public class TestCiphers { + + @Test + public void testBasicEncrypt() { + testEncryptDecrypt(null); + } + + @Test + public void testAAD() { + byte[] aad = "abcd".getBytes(StandardCharsets.UTF_8); + testEncryptDecrypt(aad); + } + + private void testEncryptDecrypt(byte[] aad) { + SecureRandom random = new SecureRandom(); + byte[] key = new byte[16]; + random.nextBytes(key); + Ciphers.AesGcmEncryptor encryptor = new Ciphers.AesGcmEncryptor(key); + byte[] plaintext = new byte[100]; + random.nextBytes(plaintext); + byte[] ciphertext = encryptor.encrypt(plaintext, aad); + + Ciphers.AesGcmDecryptor decryptor = new Ciphers.AesGcmDecryptor(key); + byte[] decryptedText = decryptor.decrypt(ciphertext, aad); + Assert.assertArrayEquals(plaintext, decryptedText); + } +} From 65d121b49846e85afa3c98aa0a23e902060da1d7 Mon Sep 17 00:00:00 2001 From: Gidon Gershinsky Date: Mon, 31 Jan 2022 13:54:10 +0200 Subject: [PATCH 09/11] post-review changes 3 --- .../apache/iceberg/encryption/Ciphers.java | 4 +++- .../iceberg/encryption/TestCiphers.java | 21 +++++++++++-------- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/core/src/main/java/org/apache/iceberg/encryption/Ciphers.java b/core/src/main/java/org/apache/iceberg/encryption/Ciphers.java index be26bb35ee57..fc63e3e66d30 100644 --- a/core/src/main/java/org/apache/iceberg/encryption/Ciphers.java +++ b/core/src/main/java/org/apache/iceberg/encryption/Ciphers.java @@ -100,7 +100,9 @@ public AesGcmDecryptor(byte[] keyBytes) { public byte[] decrypt(byte[] ciphertext, byte[] aad) { int plainTextLength = ciphertext.length - GCM_TAG_LENGTH - NONCE_LENGTH; if (plainTextLength < 1) { - throw new RuntimeException("Wrong input length " + plainTextLength); + throw new RuntimeException("Cannot decrypt cipher text of length " + ciphertext.length + + " because text must longer than GCM_TAG_LENGTH + NONCE_LENGTH bytes. Text may not be encrypted" + + " with AES GCM cipher"); } // Get the nonce from ciphertext diff --git a/core/src/test/java/org/apache/iceberg/encryption/TestCiphers.java b/core/src/test/java/org/apache/iceberg/encryption/TestCiphers.java index 8992072ec656..26aaeae0d486 100644 --- a/core/src/test/java/org/apache/iceberg/encryption/TestCiphers.java +++ b/core/src/test/java/org/apache/iceberg/encryption/TestCiphers.java @@ -39,15 +39,18 @@ public void testAAD() { private void testEncryptDecrypt(byte[] aad) { SecureRandom random = new SecureRandom(); - byte[] key = new byte[16]; - random.nextBytes(key); - Ciphers.AesGcmEncryptor encryptor = new Ciphers.AesGcmEncryptor(key); - byte[] plaintext = new byte[100]; - random.nextBytes(plaintext); - byte[] ciphertext = encryptor.encrypt(plaintext, aad); + int[] aesKeyLengthArray = {16, 24, 32}; + for (int keyLength : aesKeyLengthArray) { + byte[] key = new byte[keyLength]; + random.nextBytes(key); + Ciphers.AesGcmEncryptor encryptor = new Ciphers.AesGcmEncryptor(key); + byte[] plaintext = new byte[16]; // typically used to encrypt DEKs + random.nextBytes(plaintext); + byte[] ciphertext = encryptor.encrypt(plaintext, aad); - Ciphers.AesGcmDecryptor decryptor = new Ciphers.AesGcmDecryptor(key); - byte[] decryptedText = decryptor.decrypt(ciphertext, aad); - Assert.assertArrayEquals(plaintext, decryptedText); + Ciphers.AesGcmDecryptor decryptor = new Ciphers.AesGcmDecryptor(key); + byte[] decryptedText = decryptor.decrypt(ciphertext, aad); + Assert.assertArrayEquals("Key length " + keyLength, plaintext, decryptedText); + } } } From 1c622dc1fcd2feb55fe57ee4ae79dc1d823c249b Mon Sep 17 00:00:00 2001 From: Gidon Gershinsky Date: Wed, 16 Mar 2022 13:39:15 +0200 Subject: [PATCH 10/11] post-review changes 4 --- .../apache/iceberg/encryption/Ciphers.java | 23 ++++++++----------- .../NativeFileCryptoParameters.java | 14 ++++------- 2 files changed, 14 insertions(+), 23 deletions(-) diff --git a/core/src/main/java/org/apache/iceberg/encryption/Ciphers.java b/core/src/main/java/org/apache/iceberg/encryption/Ciphers.java index fc63e3e66d30..6e1d7a04b6ed 100644 --- a/core/src/main/java/org/apache/iceberg/encryption/Ciphers.java +++ b/core/src/main/java/org/apache/iceberg/encryption/Ciphers.java @@ -25,6 +25,7 @@ import javax.crypto.Cipher; import javax.crypto.spec.GCMParameterSpec; import javax.crypto.spec.SecretKeySpec; +import org.apache.iceberg.relocated.com.google.common.base.Preconditions; public class Ciphers { private static final int NONCE_LENGTH = 12; @@ -38,10 +39,8 @@ public static class AesGcmEncryptor { public AesGcmEncryptor(byte[] keyBytes) { int keyLength = keyBytes.length; - if (!(keyLength == 16 || keyLength == 24 || keyLength == 32)) { - throw new IllegalArgumentException("Cannot use a key of length " + keyLength + - " because AES only allows 16, 24 or 32 bytes"); - } + Preconditions.checkArgument((keyLength == 16 || keyLength == 24 || keyLength == 32), + "Cannot use a key of length " + keyLength + " because AES only allows 16, 24 or 32 bytes"); this.aesKey = new SecretKeySpec(keyBytes, "AES"); try { @@ -83,11 +82,8 @@ public static class AesGcmDecryptor { public AesGcmDecryptor(byte[] keyBytes) { int keyLength = keyBytes.length; - if (!(keyLength == 16 || keyLength == 24 || keyLength == 32)) { - throw new IllegalArgumentException("Cannot use a key of length " + keyLength + - " because AES only allows 16, 24 or 32 bytes"); - } - + Preconditions.checkArgument((keyLength == 16 || keyLength == 24 || keyLength == 32), + "Cannot use a key of length " + keyLength + " because AES only allows 16, 24 or 32 bytes"); this.aesKey = new SecretKeySpec(keyBytes, "AES"); try { @@ -99,11 +95,10 @@ public AesGcmDecryptor(byte[] keyBytes) { public byte[] decrypt(byte[] ciphertext, byte[] aad) { int plainTextLength = ciphertext.length - GCM_TAG_LENGTH - NONCE_LENGTH; - if (plainTextLength < 1) { - throw new RuntimeException("Cannot decrypt cipher text of length " + ciphertext.length + - " because text must longer than GCM_TAG_LENGTH + NONCE_LENGTH bytes. Text may not be encrypted" + - " with AES GCM cipher"); - } + Preconditions.checkState(plainTextLength >= 1, + "Cannot decrypt cipher text of length " + ciphertext.length + + " because text must longer than GCM_TAG_LENGTH + NONCE_LENGTH bytes. Text may not be encrypted" + + " with AES GCM cipher"); // Get the nonce from ciphertext byte[] nonce = new byte[NONCE_LENGTH]; diff --git a/core/src/main/java/org/apache/iceberg/encryption/NativeFileCryptoParameters.java b/core/src/main/java/org/apache/iceberg/encryption/NativeFileCryptoParameters.java index 4b04b8bd6749..c19ab2fcd759 100644 --- a/core/src/main/java/org/apache/iceberg/encryption/NativeFileCryptoParameters.java +++ b/core/src/main/java/org/apache/iceberg/encryption/NativeFileCryptoParameters.java @@ -29,9 +29,9 @@ */ public class NativeFileCryptoParameters { private ByteBuffer fileKey; - private String fileEncryptionAlgorithm; + private EncryptionAlgorithm fileEncryptionAlgorithm; - private NativeFileCryptoParameters(ByteBuffer fileKey, String fileEncryptionAlgorithm) { + private NativeFileCryptoParameters(ByteBuffer fileKey, EncryptionAlgorithm fileEncryptionAlgorithm) { Preconditions.checkState(fileKey != null, "File encryption key is not supplied"); this.fileKey = fileKey; this.fileEncryptionAlgorithm = fileEncryptionAlgorithm; @@ -48,13 +48,13 @@ public static Builder create(ByteBuffer fileKey) { public static class Builder { private ByteBuffer fileKey; - private String fileEncryptionAlgorithm; + private EncryptionAlgorithm fileEncryptionAlgorithm; private Builder(ByteBuffer fileKey) { this.fileKey = fileKey; } - public Builder encryptionAlgorithm(String encryptionAlgorithm) { + public Builder encryptionAlgorithm(EncryptionAlgorithm encryptionAlgorithm) { this.fileEncryptionAlgorithm = encryptionAlgorithm; return this; } @@ -62,17 +62,13 @@ public Builder encryptionAlgorithm(String encryptionAlgorithm) { public NativeFileCryptoParameters build() { return new NativeFileCryptoParameters(fileKey, fileEncryptionAlgorithm); } - - // TODO add back column encryption keys - - // TODO add back AAD prefix (cryptographic file identity) } public ByteBuffer fileKey() { return fileKey; } - public String encryptionAlgorithm() { + public EncryptionAlgorithm encryptionAlgorithm() { return fileEncryptionAlgorithm; } } From b323c5f37ff0f29599de0832fa96a42633d4f52a Mon Sep 17 00:00:00 2001 From: Gidon Gershinsky Date: Wed, 23 Mar 2022 10:42:48 +0200 Subject: [PATCH 11/11] post-review changes 5 --- .../src/main/java/org/apache/iceberg/encryption/Ciphers.java | 5 ++++- .../org/apache/iceberg/encryption/EncryptionAlgorithm.java | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/org/apache/iceberg/encryption/Ciphers.java b/core/src/main/java/org/apache/iceberg/encryption/Ciphers.java index 6e1d7a04b6ed..11c09543f4ee 100644 --- a/core/src/main/java/org/apache/iceberg/encryption/Ciphers.java +++ b/core/src/main/java/org/apache/iceberg/encryption/Ciphers.java @@ -38,6 +38,7 @@ public static class AesGcmEncryptor { private final SecureRandom randomGenerator; public AesGcmEncryptor(byte[] keyBytes) { + Preconditions.checkArgument(keyBytes != null, "Key can't be null"); int keyLength = keyBytes.length; Preconditions.checkArgument((keyLength == 16 || keyLength == 24 || keyLength == 32), "Cannot use a key of length " + keyLength + " because AES only allows 16, 24 or 32 bytes"); @@ -81,6 +82,7 @@ public static class AesGcmDecryptor { private final Cipher cipher; public AesGcmDecryptor(byte[] keyBytes) { + Preconditions.checkArgument(keyBytes != null, "Key can't be null"); int keyLength = keyBytes.length; Preconditions.checkArgument((keyLength == 16 || keyLength == 24 || keyLength == 32), "Cannot use a key of length " + keyLength + " because AES only allows 16, 24 or 32 bytes"); @@ -114,7 +116,8 @@ public byte[] decrypt(byte[] ciphertext, byte[] aad) { } cipher.doFinal(ciphertext, NONCE_LENGTH, inputLength, plainText, 0); } catch (AEADBadTagException e) { - throw new RuntimeException("GCM tag check failed", e); + throw new RuntimeException("GCM tag check failed. Possible reasons: wrong decryption key; or corrupt/tampered" + + "data. AES GCM doesn't differentiate between these two.. ", e); } catch (GeneralSecurityException e) { throw new RuntimeException("Failed to decrypt", e); } diff --git a/core/src/main/java/org/apache/iceberg/encryption/EncryptionAlgorithm.java b/core/src/main/java/org/apache/iceberg/encryption/EncryptionAlgorithm.java index 0ada5900e4b6..650958c5a3b7 100644 --- a/core/src/main/java/org/apache/iceberg/encryption/EncryptionAlgorithm.java +++ b/core/src/main/java/org/apache/iceberg/encryption/EncryptionAlgorithm.java @@ -46,7 +46,7 @@ public enum EncryptionAlgorithm { * A combination of GCM and CTR that can be used for file types like Parquet, * so that all modules except pages are encrypted by GCM to ensure integrity, * and CTR is used for efficient encryption of bulk data. - * The tradeoff is that attackers would be able to tamper page data. + * The tradeoff is that attackers would be able to tamper page data encrypted with CTR. */ AES_GCM_CTR }