From 882897cf920f347be468f7407dbfe93a18f8ae7f Mon Sep 17 00:00:00 2001 From: Syed Shameerur Rahman Date: Tue, 8 Oct 2024 14:13:54 +0530 Subject: [PATCH 1/6] HADOOP-18708: Support S3 Client Side Encryption(CSE) With AWS SDK V2 --- .../apache/hadoop/fs/MultipartUploader.java | 4 +- .../fs/impl/AbstractMultipartUploader.java | 2 +- .../fs/impl/FileSystemMultipartUploader.java | 2 +- ...AbstractContractMultipartUploaderTest.java | 36 ++- hadoop-project/pom.xml | 12 + hadoop-tools/hadoop-aws/pom.xml | 15 + .../org/apache/hadoop/fs/s3a/Constants.java | 34 ++ .../hadoop/fs/s3a/DefaultS3ClientFactory.java | 19 +- .../org/apache/hadoop/fs/s3a/Listing.java | 183 ++++++++++- .../hadoop/fs/s3a/S3ABlockOutputStream.java | 14 +- .../apache/hadoop/fs/s3a/S3AFileSystem.java | 141 +++++---- .../apache/hadoop/fs/s3a/S3AInputStream.java | 6 +- .../org/apache/hadoop/fs/s3a/S3AUtils.java | 9 +- .../apache/hadoop/fs/s3a/S3ClientFactory.java | 50 +++ .../hadoop/fs/s3a/WriteOperationHelper.java | 3 + .../apache/hadoop/fs/s3a/WriteOperations.java | 2 + .../hadoop/fs/s3a/api/RequestFactory.java | 2 + .../fs/s3a/commit/impl/CommitOperations.java | 8 +- .../apache/hadoop/fs/s3a/impl/AWSHeaders.java | 5 + .../fs/s3a/impl/BaseS3AFileSystemHandler.java | 163 ++++++++++ .../hadoop/fs/s3a/impl/CSEMaterials.java | 132 ++++++++ .../fs/s3a/impl/CSES3AFileSystemHandler.java | 166 ++++++++++ .../apache/hadoop/fs/s3a/impl/CSEUtils.java | 196 ++++++++++++ .../CSEV1CompatibleS3AFileSystemHandler.java | 132 ++++++++ .../hadoop/fs/s3a/impl/ClientManager.java | 2 + .../hadoop/fs/s3a/impl/ClientManagerImpl.java | 40 +++ .../s3a/impl/EncryptionS3ClientFactory.java | 294 ++++++++++++++++++ .../hadoop/fs/s3a/impl/HeaderProcessing.java | 13 - .../s3a/impl/ListingOperationCallbacks.java | 11 + .../fs/s3a/impl/RequestFactoryImpl.java | 5 + .../fs/s3a/impl/S3AFileSystemHandler.java | 125 ++++++++ .../fs/s3a/impl/S3AMultipartUploader.java | 3 +- .../hadoop/fs/s3a/impl/S3AStoreImpl.java | 5 + .../markdown/tools/hadoop-aws/encryption.md | 41 ++- .../apache/hadoop/fs/s3a/CustomKeyring.java | 72 +++++ .../fs/s3a/ITestS3AClientSideEncryption.java | 225 +++++++++++++- .../ITestS3AClientSideEncryptionCustom.java | 84 +++++ .../hadoop/fs/s3a/ITestS3AConfiguration.java | 21 +- .../hadoop/fs/s3a/ITestS3ADelayedFNF.java | 11 +- ...TestS3AEncryptionSSEKMSUserDefinedKey.java | 4 +- .../hadoop/fs/s3a/ITestS3AEndpointRegion.java | 4 +- .../fs/s3a/ITestS3AFailureHandling.java | 14 +- .../hadoop/fs/s3a/MultipartTestUtils.java | 2 +- .../fs/s3a/TestS3ABlockOutputStream.java | 4 +- .../integration/ITestS3ACommitterMRJob.java | 4 + .../ITestS3AFileContextStatistics.java | 30 +- .../fs/s3a/impl/ITestConnectionTimeouts.java | 10 +- .../hadoop/fs/s3a/impl/TestClientManager.java | 1 + .../fs/s3a/impl/TestRequestFactory.java | 14 +- .../MinimalListingOperationCallbacks.java | 8 + 50 files changed, 2209 insertions(+), 174 deletions(-) create mode 100644 hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/BaseS3AFileSystemHandler.java create mode 100644 hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/CSEMaterials.java create mode 100644 hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/CSES3AFileSystemHandler.java create mode 100644 hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/CSEUtils.java create mode 100644 hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/CSEV1CompatibleS3AFileSystemHandler.java create mode 100644 hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/EncryptionS3ClientFactory.java create mode 100644 hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/S3AFileSystemHandler.java create mode 100644 hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/CustomKeyring.java create mode 100644 hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AClientSideEncryptionCustom.java diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/MultipartUploader.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/MultipartUploader.java index 5e4eda26c7f1db..c41a03b94fdfbe 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/MultipartUploader.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/MultipartUploader.java @@ -57,6 +57,7 @@ CompletableFuture startUpload(Path filePath) * It is possible to have parts uploaded in any order (or in parallel). * @param uploadId Identifier from {@link #startUpload(Path)}. * @param partNumber Index of the part relative to others. + * @param isLastPart is the part the last part of the upload? * @param filePath Target path for upload (as {@link #startUpload(Path)}). * @param inputStream Data for this part. Implementations MUST close this * stream after reading in the data. @@ -67,6 +68,7 @@ CompletableFuture startUpload(Path filePath) CompletableFuture putPart( UploadHandle uploadId, int partNumber, + boolean isLastPart, Path filePath, InputStream inputStream, long lengthInBytes) @@ -77,7 +79,7 @@ CompletableFuture putPart( * @param uploadId Identifier from {@link #startUpload(Path)}. * @param filePath Target path for upload (as {@link #startUpload(Path)}. * @param handles non-empty map of part number to part handle. - * from {@link #putPart(UploadHandle, int, Path, InputStream, long)}. + * from {@link #putPart(UploadHandle, int, boolean, Path, InputStream, long)}. * @return unique PathHandle identifier for the uploaded file. * @throws IOException IO failure */ diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/AbstractMultipartUploader.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/AbstractMultipartUploader.java index f9ae9f55cc17f1..f3502600a84fde 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/AbstractMultipartUploader.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/AbstractMultipartUploader.java @@ -101,7 +101,7 @@ protected void checkPartHandles(Map partHandles) { /** * Check all the arguments to the - * {@link MultipartUploader#putPart(UploadHandle, int, Path, InputStream, long)} + * {@link MultipartUploader#putPart(UploadHandle, int, boolean, Path, InputStream, long)} * operation. * @param filePath Target path for upload (as {@link #startUpload(Path)}). * @param inputStream Data for this part. Implementations MUST close this diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/FileSystemMultipartUploader.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/FileSystemMultipartUploader.java index 28a4bce0489cd5..09d9079cff3deb 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/FileSystemMultipartUploader.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/FileSystemMultipartUploader.java @@ -111,7 +111,7 @@ public CompletableFuture startUpload(Path filePath) @Override public CompletableFuture putPart(UploadHandle uploadId, - int partNumber, Path filePath, + int partNumber, boolean isLastPart, Path filePath, InputStream inputStream, long lengthInBytes) throws IOException { diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractMultipartUploaderTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractMultipartUploaderTest.java index 7420b47a984958..16482915bdf7f4 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractMultipartUploaderTest.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractMultipartUploaderTest.java @@ -249,7 +249,7 @@ public void testSingleUpload() throws Exception { // was interpreted as an inconsistent write. MultipartUploader completer = uploader0; // and upload with uploader 1 to validate cross-uploader uploads - PartHandle partHandle = putPart(file, uploadHandle, 1, payload); + PartHandle partHandle = putPart(file, uploadHandle, 1, true, payload); partHandles.put(1, partHandle); PathHandle fd = complete(completer, uploadHandle, file, partHandles); @@ -317,12 +317,13 @@ protected PartHandle buildAndPutPart( final Path file, final UploadHandle uploadHandle, final int index, + final boolean isLastPart, final MessageDigest origDigest) throws IOException { byte[] payload = generatePayload(index); if (origDigest != null) { origDigest.update(payload); } - return putPart(file, uploadHandle, index, payload); + return putPart(file, uploadHandle, index, isLastPart, payload); } /** @@ -331,6 +332,7 @@ protected PartHandle buildAndPutPart( * @param file destination * @param uploadHandle handle * @param index index of part + * @param isLastPart is last part of the upload ? * @param payload byte array of payload * @return the part handle * @throws IOException IO failure. @@ -338,6 +340,7 @@ protected PartHandle buildAndPutPart( protected PartHandle putPart(final Path file, final UploadHandle uploadHandle, final int index, + final boolean isLastPart, final byte[] payload) throws IOException { ContractTestUtils.NanoTimer timer = new ContractTestUtils.NanoTimer(); PartHandle partHandle; @@ -347,7 +350,7 @@ protected PartHandle putPart(final Path file, payload.length, file)) { partHandle = awaitFuture(getUploader(index) - .putPart(uploadHandle, index, file, + .putPart(uploadHandle, index, isLastPart, file, new ByteArrayInputStream(payload), payload.length)); } @@ -488,7 +491,7 @@ public void testMultipartUpload() throws Exception { MessageDigest origDigest = DigestUtils.getMd5Digest(); int payloadCount = getTestPayloadCount(); for (int i = 1; i <= payloadCount; ++i) { - PartHandle partHandle = buildAndPutPart(file, uploadHandle, i, + PartHandle partHandle = buildAndPutPart(file, uploadHandle, i, i == payloadCount, origDigest); partHandles.put(i, partHandle); } @@ -515,7 +518,7 @@ public void testMultipartUploadEmptyPart() throws Exception { origDigest.update(payload); InputStream is = new ByteArrayInputStream(payload); PartHandle partHandle = awaitFuture( - uploader.putPart(uploadHandle, 1, file, is, payload.length)); + uploader.putPart(uploadHandle, 1, true, file, is, payload.length)); partHandles.put(1, partHandle); completeUpload(file, uploadHandle, partHandles, origDigest, 0); } @@ -530,7 +533,7 @@ public void testUploadEmptyBlock() throws Exception { Path file = methodPath(); UploadHandle uploadHandle = startUpload(file); Map partHandles = new HashMap<>(); - partHandles.put(1, putPart(file, uploadHandle, 1, new byte[0])); + partHandles.put(1, putPart(file, uploadHandle, 1, true, new byte[0])); completeUpload(file, uploadHandle, partHandles, null, 0); } @@ -550,7 +553,8 @@ public void testMultipartUploadReverseOrder() throws Exception { origDigest.update(payload); } for (int i = payloadCount; i > 0; --i) { - partHandles.put(i, buildAndPutPart(file, uploadHandle, i, null)); + partHandles.put(i, buildAndPutPart(file, uploadHandle, i, i == payloadCount, + null)); } completeUpload(file, uploadHandle, partHandles, origDigest, payloadCount * partSizeInBytes()); @@ -574,7 +578,8 @@ public void testMultipartUploadReverseOrderNonContiguousPartNumbers() } Map partHandles = new HashMap<>(); for (int i = payloadCount; i > 0; i -= 2) { - partHandles.put(i, buildAndPutPart(file, uploadHandle, i, null)); + partHandles.put(i, buildAndPutPart(file, uploadHandle, i, i == payloadCount, + null)); } completeUpload(file, uploadHandle, partHandles, origDigest, getTestPayloadCount() * partSizeInBytes()); @@ -591,7 +596,7 @@ public void testMultipartUploadAbort() throws Exception { UploadHandle uploadHandle = startUpload(file); Map partHandles = new HashMap<>(); for (int i = 12; i > 10; i--) { - partHandles.put(i, buildAndPutPart(file, uploadHandle, i, null)); + partHandles.put(i, buildAndPutPart(file, uploadHandle, i, i == 12, null)); } abortUpload(uploadHandle, file); @@ -601,7 +606,7 @@ public void testMultipartUploadAbort() throws Exception { intercept(IOException.class, () -> awaitFuture( - uploader0.putPart(uploadHandle, 49, file, is, len))); + uploader0.putPart(uploadHandle, 49, true, file, is, len))); intercept(IOException.class, () -> complete(uploader0, uploadHandle, file, partHandles)); @@ -701,7 +706,8 @@ public void testPutPartEmptyUploadID() throws Exception { byte[] payload = generatePayload(1); InputStream is = new ByteArrayInputStream(payload); intercept(IllegalArgumentException.class, - () -> uploader0.putPart(emptyHandle, 1, dest, is, payload.length)); + () -> uploader0.putPart(emptyHandle, 1, true, dest, is, + payload.length)); } /** @@ -715,7 +721,7 @@ public void testCompleteEmptyUploadID() throws Exception { UploadHandle emptyHandle = BBUploadHandle.from(ByteBuffer.wrap(new byte[0])); Map partHandles = new HashMap<>(); - PartHandle partHandle = putPart(dest, realHandle, 1, + PartHandle partHandle = putPart(dest, realHandle, 1, true, generatePayload(1, SMALL_FILE)); partHandles.put(1, partHandle); @@ -743,7 +749,7 @@ public void testDirectoryInTheWay() throws Exception { UploadHandle uploadHandle = startUpload(file); Map partHandles = new HashMap<>(); int size = SMALL_FILE; - PartHandle partHandle = putPart(file, uploadHandle, 1, + PartHandle partHandle = putPart(file, uploadHandle, 1, true, generatePayload(1, size)); partHandles.put(1, partHandle); @@ -802,10 +808,10 @@ public void testConcurrentUploads() throws Throwable { assertNotEquals("Upload handles match", upload1, upload2); // put part 1 - partHandles1.put(partId1, putPart(file, upload1, partId1, payload1)); + partHandles1.put(partId1, putPart(file, upload1, partId1, false, payload1)); // put part2 - partHandles2.put(partId2, putPart(file, upload2, partId2, payload2)); + partHandles2.put(partId2, putPart(file, upload2, partId2, true, payload2)); // complete part u1. expect its size and digest to // be as expected. diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index faf76544aff5b9..8c36792d1d5991 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -205,6 +205,7 @@ 900 1.12.720 2.25.53 + 3.1.1 1.0.1 2.7.1 1.11.2 @@ -1180,6 +1181,17 @@ + + software.amazon.encryption.s3 + amazon-s3-encryption-client-java + ${amazon-s3-encryption-client-java.version} + + + * + * + + + org.apache.mina mina-core diff --git a/hadoop-tools/hadoop-aws/pom.xml b/hadoop-tools/hadoop-aws/pom.xml index 997f63b0dc15f6..f7119b62e19899 100644 --- a/hadoop-tools/hadoop-aws/pom.xml +++ b/hadoop-tools/hadoop-aws/pom.xml @@ -466,6 +466,16 @@ org.apache.hadoop.mapred.** + + false + Restrict encryption client imports to encryption client factory + + org.apache.hadoop.fs.s3a.impl.EncryptionS3ClientFactory + + + software.amazon.encryption.s3.** + + @@ -510,6 +520,11 @@ bundle compile + + software.amazon.encryption.s3 + amazon-s3-encryption-client-java + provided + org.assertj assertj-core diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java index 67854e65720fdd..5d98494a278e81 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java @@ -766,6 +766,40 @@ private Constants() { public static final String S3_ENCRYPTION_CONTEXT = "fs.s3a.encryption.context"; + /** + * Client side encryption (CSE-CUSTOM) with custom cryptographic material manager class name. + * Custom keyring class name for CSE-KMS. + * value:{@value} + */ + public static final String S3_ENCRYPTION_CSE_CUSTOM_KEYRING_CLASS_NAME = + "fs.s3a.encryption.cse.custom.keyring.class.name"; + + /** + * Config to provide backward compatibility with V1 encryption client. + * Enabling this configuration will invoke the followings + * 1. Unencrypted s3 objects will be read using unecrypted/base s3 client when CSE is enabled. + * 2. Size of encrypted object will be calculated using ranged S3 calls. + * 3. While listing s3 objects, encryption metadata file with suffix + * {@link #S3_ENCRYPTION_CSE_INSTRUCTION_FILE_SUFFIX} will be skipped. + * This is to provide backward compatibility with V1 client. + * value:{@value} + */ + public static final String S3_ENCRYPTION_CSE_V1_COMPATIBILITY_ENABLED = + "fs.s3a.encryption.cse.v1.compatibility.enabled"; + + /** + * Default value : {@value}. + */ + public static final boolean S3_ENCRYPTION_CSE_V1_COMPATIBILITY_ENABLED_DEFAULT = false; + + /** + * Suffix of instruction file : {@value}. + */ + public static final String S3_ENCRYPTION_CSE_INSTRUCTION_FILE_SUFFIX = ".instruction"; + + + + /** * List of custom Signers. The signer class will be loaded, and the signer * name will be associated with this signer class in the S3 SDK. diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/DefaultS3ClientFactory.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/DefaultS3ClientFactory.java index 0b18ce999f813e..96a57ed087ee18 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/DefaultS3ClientFactory.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/DefaultS3ClientFactory.java @@ -41,6 +41,7 @@ import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.s3accessgrants.plugin.S3AccessGrantsPlugin; import software.amazon.awssdk.services.s3.S3AsyncClient; +import software.amazon.awssdk.services.s3.S3AsyncClientBuilder; import software.amazon.awssdk.services.s3.S3BaseClientBuilder; import software.amazon.awssdk.services.s3.S3Client; import software.amazon.awssdk.services.s3.S3Configuration; @@ -160,11 +161,17 @@ public S3AsyncClient createS3AsyncClient( .thresholdInBytes(parameters.getMultiPartThreshold()) .build(); - return configureClientBuilder(S3AsyncClient.builder(), parameters, conf, bucket) - .httpClientBuilder(httpClientBuilder) - .multipartConfiguration(multipartConfiguration) - .multipartEnabled(parameters.isMultipartCopy()) - .build(); + S3AsyncClientBuilder s3AsyncClientBuilder = + configureClientBuilder(S3AsyncClient.builder(), parameters, conf, bucket) + .httpClientBuilder(httpClientBuilder); + + // TODO: Enable multi part upload with cse once it is available. + if (!parameters.isClientSideEncryptionEnabled()) { + s3AsyncClientBuilder.multipartConfiguration(multipartConfiguration) + .multipartEnabled(parameters.isMultipartCopy()); + } + + return s3AsyncClientBuilder.build(); } @Override @@ -373,7 +380,7 @@ private , ClientT> void * @param conf config to build the URI from. * @return an endpoint uri */ - private static URI getS3Endpoint(String endpoint, final Configuration conf) { + public static URI getS3Endpoint(String endpoint, final Configuration conf) { boolean secureConnections = conf.getBoolean(SECURE_CONNECTIONS, DEFAULT_SECURE_CONNECTIONS); diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Listing.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Listing.java index e0868a2e130874..c4319669d6527a 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Listing.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Listing.java @@ -30,6 +30,7 @@ import org.apache.hadoop.fs.RemoteIterator; import org.apache.hadoop.fs.s3a.impl.AbstractStoreOperation; import org.apache.hadoop.fs.s3a.impl.ListingOperationCallbacks; +import org.apache.hadoop.fs.s3a.impl.S3AFileSystemHandler; import org.apache.hadoop.fs.s3a.impl.StoreContext; import org.apache.hadoop.fs.statistics.IOStatistics; import org.apache.hadoop.fs.statistics.IOStatisticsAggregator; @@ -58,6 +59,7 @@ import static org.apache.hadoop.fs.s3a.S3AUtils.objectRepresentsDirectory; import static org.apache.hadoop.fs.s3a.S3AUtils.stringify; import static org.apache.hadoop.fs.s3a.auth.RoleModel.pathToKey; +import static org.apache.hadoop.fs.s3a.impl.CSEUtils.isCSEInstructionFile; import static org.apache.hadoop.fs.statistics.StoreStatisticNames.OBJECT_CONTINUE_LIST_REQUEST; import static org.apache.hadoop.fs.statistics.StoreStatisticNames.OBJECT_LIST_REQUEST; import static org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding.iostatisticsStore; @@ -75,18 +77,18 @@ public class Listing extends AbstractStoreOperation { private static final Logger LOG = S3AFileSystem.LOG; - private final boolean isCSEEnabled; + private final S3AFileSystemHandler handler; - static final FileStatusAcceptor ACCEPT_ALL_BUT_S3N = + public static final FileStatusAcceptor ACCEPT_ALL_BUT_S3N = new AcceptAllButS3nDirs(); private final ListingOperationCallbacks listingOperationCallbacks; public Listing(ListingOperationCallbacks listingOperationCallbacks, - StoreContext storeContext) { + StoreContext storeContext, S3AFileSystemHandler handler) { super(storeContext); this.listingOperationCallbacks = listingOperationCallbacks; - this.isCSEEnabled = storeContext.isCSEEnabled(); + this.handler = handler; } /** @@ -237,7 +239,7 @@ public RemoteIterator getLocatedFileStatusIteratorForDir( listingOperationCallbacks .createListObjectsRequest(key, "/", span), filter, - new AcceptAllButSelfAndS3nDirs(dir), + handler.getFileStatusAcceptor(dir, false), span)); } @@ -266,7 +268,7 @@ public RemoteIterator getLocatedFileStatusIteratorForDir( path, request, ACCEPT_ALL, - new AcceptAllButSelfAndS3nDirs(path), + handler.getFileStatusAcceptor(path, false), span); } @@ -281,7 +283,7 @@ public S3ListRequest createListObjectsRequest(String key, * Interface to implement the logic deciding whether to accept a s3Object * entry or path as a valid file or directory. */ - interface FileStatusAcceptor { + public interface FileStatusAcceptor { /** * Predicate to decide whether or not to accept a s3Object entry. @@ -447,7 +449,7 @@ private boolean requestNextBatch() throws IOException { * @param objects the next object listing * @return true if this added any entries after filtering */ - private boolean buildNextStatusBatch(S3ListResult objects) { + private boolean buildNextStatusBatch(S3ListResult objects) throws IOException { // counters for debug logs int added = 0, ignored = 0; // list to fill in with results. Initial size will be list maximum. @@ -464,9 +466,10 @@ private boolean buildNextStatusBatch(S3ListResult objects) { // Skip over keys that are ourselves and old S3N _$folder$ files if (acceptor.accept(keyPath, s3Object) && filter.accept(keyPath)) { S3AFileStatus status = createFileStatus(keyPath, s3Object, - listingOperationCallbacks.getDefaultBlockSize(keyPath), - getStoreContext().getUsername(), - s3Object.eTag(), null, isCSEEnabled); + listingOperationCallbacks.getDefaultBlockSize(keyPath), + getStoreContext().getUsername(), + s3Object.eTag(), null, + listingOperationCallbacks.getObjectSize(s3Object)); LOG.debug("Adding: {}", status); stats.add(status); added++; @@ -724,7 +727,7 @@ public void close() { * Accept all entries except the base path and those which map to S3N * pseudo directory markers. */ - static class AcceptFilesOnly implements FileStatusAcceptor { + public static class AcceptFilesOnly implements FileStatusAcceptor { private final Path qualifiedPath; public AcceptFilesOnly(Path qualifiedPath) { @@ -763,6 +766,61 @@ public boolean accept(FileStatus status) { } } + /** + * Accept all entries except the base path and those which map to S3N + * pseudo directory markers and CSE instruction file. + */ + public static class AcceptFilesOnlyExceptCSEInstructionFile implements FileStatusAcceptor { + private final Path qualifiedPath; + + /** + * Constructor. + * @param qualifiedPath an already-qualified path. + */ + public AcceptFilesOnlyExceptCSEInstructionFile(Path qualifiedPath) { + this.qualifiedPath = qualifiedPath; + } + + /** + * Reject a s3Object entry if the key path is the qualified Path, or + * it ends with {@code "_$folder$"} or {@code ".instruction"}. + * @param keyPath key path of the entry + * @param s3Object s3Object entry + * @return true if the entry is accepted (i.e. that a status entry + * should be generated. + */ + @Override + public boolean accept(Path keyPath, S3Object s3Object) { + return !keyPath.equals(qualifiedPath) + && !s3Object.key().endsWith(S3N_FOLDER_SUFFIX) + && !objectRepresentsDirectory(s3Object.key()) + && !isCSEInstructionFile(s3Object.key()); + } + + /** + * Accept no directory paths. + * @param keyPath qualified path to the entry + * @param prefix common prefix in listing. + * @return false, always. + */ + @Override + public boolean accept(Path keyPath, String prefix) { + return false; + } + + /** + * Reject if the file status is not a file or is a CSE instruction file. + * @param status file status containing file path information + * @return true if the entry is accepted (i.e. that a status entry + * should be generated. + */ + @Override + public boolean accept(FileStatus status) { + return (status != null) && status.isFile() + && !isCSEInstructionFile(status.getPath().toString()); + } + } + /** * Accept all entries except those which map to S3N pseudo directory markers. */ @@ -782,6 +840,47 @@ public boolean accept(FileStatus status) { } + /** + * Accept all entries except those which map to S3N pseudo directory markers and CSE instruction + * file. + */ + public static class AcceptAllButS3nDirsAndCSEInstructionFile implements FileStatusAcceptor { + + /** + * Reject a s3Object entry if the key ends with {@code "_$folder$"} or {@code ".instruction"}. + * @param keyPath key path of the entry + * @param s3Object s3Object entry + * @return true if the entry is accepted (i.e. that a status entry + * should be generated. + */ + public boolean accept(Path keyPath, S3Object s3Object) { + return !s3Object.key().endsWith(S3N_FOLDER_SUFFIX) && + !isCSEInstructionFile(s3Object.key()); + } + + /** + * Reject if the key ends with {@code "_$folder$"} or {@code ".instruction"}. + * @param keyPath qualified path to the entry + * @param prefix the prefix + * @return true if the entry is accepted + */ + public boolean accept(Path keyPath, String prefix) { + return !keyPath.toString().endsWith(S3N_FOLDER_SUFFIX) && + !isCSEInstructionFile(keyPath.toString()); + } + + /** + * Reject if the file status is a CSE instruction file or ends with {@code "_$folder$"}. + * @param status file status containing file path information + * @return true if the entry is accepted (i.e. that a status entry + * should be generated. + */ + public boolean accept(FileStatus status) { + return !status.getPath().toString().endsWith(S3N_FOLDER_SUFFIX) && + isCSEInstructionFile(status.getPath().toString()); + } + } + /** * Accept all entries except the base path and those which map to S3N * pseudo directory markers. @@ -831,6 +930,66 @@ public boolean accept(FileStatus status) { } } + /** + * Accept all entries except the base path, those which map to S3N pseudo directory markers + * and files which ends with CSE instruction file. + */ + public static class AcceptAllButSelfAndS3nDirsAndCSEInstructionFile + implements FileStatusAcceptor { + + /** Base path. */ + private final Path qualifiedPath; + + /** + * Constructor. + * @param qualifiedPath an already-qualified path. + */ + public AcceptAllButSelfAndS3nDirsAndCSEInstructionFile(Path qualifiedPath) { + this.qualifiedPath = qualifiedPath; + } + + /** + * Reject a s3Object entry if the key path is the qualified Path, or + * it ends with {@code "_$folder$"}. or ends with {@code ".instruction"}. + * @param keyPath key path of the entry + * @param s3Object s3Object entry + * @return true if the entry is accepted (i.e. that a status entry + * should be generated.) + */ + @Override + public boolean accept(Path keyPath, S3Object s3Object) { + return !keyPath.equals(qualifiedPath) && + !s3Object.key().endsWith(S3N_FOLDER_SUFFIX) && + !isCSEInstructionFile(s3Object.key()); + } + + /** + * Accept all prefixes except the one for the base path, "self" and CSE instruction file. + * @param keyPath qualified path to the entry + * @param prefix common prefix in listing. + * @return true if the entry is accepted (i.e. that a status entry + * should be generated. + */ + @Override + public boolean accept(Path keyPath, String prefix) { + return !keyPath.equals(qualifiedPath) && + !isCSEInstructionFile(keyPath.toString()); + } + + /** + * Reject if the file status is the base path, a CSE instruction file or ends + * with {@code "_$folder$"}. + * @param status file status containing file path information + * @return true if the entry is accepted (i.e. that a status entry + * should be generated. + */ + @Override + public boolean accept(FileStatus status) { + return (status != null) && !status.getPath().equals(qualifiedPath) && + !isCSEInstructionFile(status.getPath().toString()); + } + } + @SuppressWarnings("unchecked") public static RemoteIterator toLocatedFileStatusIterator( RemoteIterator iterator) { diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ABlockOutputStream.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ABlockOutputStream.java index 741a78a0537f2b..7b249a11c07d4f 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ABlockOutputStream.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ABlockOutputStream.java @@ -39,6 +39,7 @@ import javax.annotation.Nonnull; +import software.amazon.awssdk.core.exception.SdkException; import software.amazon.awssdk.core.sync.RequestBody; import software.amazon.awssdk.services.s3.model.CompletedPart; import software.amazon.awssdk.services.s3.model.PutObjectRequest; @@ -76,6 +77,7 @@ import org.apache.hadoop.util.Progressable; import static java.util.Objects.requireNonNull; +import static org.apache.hadoop.fs.s3a.S3AUtils.translateException; import static org.apache.hadoop.fs.s3a.Statistic.*; import static org.apache.hadoop.fs.s3a.impl.HeaderProcessing.CONTENT_TYPE_OCTET_STREAM; import static org.apache.hadoop.fs.s3a.impl.ProgressListenerEvent.*; @@ -1002,11 +1004,19 @@ private void uploadBlockAsync(final S3ADataBlocks.DataBlock block, uploadData.getSize(), CONTENT_TYPE_OCTET_STREAM); - request = writeOperationHelper.newUploadPartRequestBuilder( + UploadPartRequest.Builder requestBuilder = writeOperationHelper.newUploadPartRequestBuilder( key, uploadId, currentPartNumber, - size).build(); + isLast, + size); + request = requestBuilder.build(); + } catch (SdkException aws) { + // catch and translate + IOException e = translateException("upload", key, aws); + // failure to start the upload. + noteUploadFailure(e); + throw e; } catch (IOException e) { // failure to prepare the upload. noteUploadFailure(e); diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java index 3ab19dfa1914f6..1c81a3ff1b0330 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java @@ -116,7 +116,7 @@ import org.apache.hadoop.fs.s3a.auth.delegation.DelegationTokenProvider; import org.apache.hadoop.fs.s3a.commit.magic.InMemoryMagicCommitTracker; import org.apache.hadoop.fs.s3a.impl.AWSCannedACL; -import org.apache.hadoop.fs.s3a.impl.AWSHeaders; +import org.apache.hadoop.fs.s3a.impl.BaseS3AFileSystemHandler; import org.apache.hadoop.fs.s3a.impl.BulkDeleteOperation; import org.apache.hadoop.fs.s3a.impl.BulkDeleteOperationCallbacksImpl; import org.apache.hadoop.fs.s3a.impl.ChangeDetectionPolicy; @@ -126,6 +126,10 @@ import org.apache.hadoop.fs.s3a.impl.ContextAccessors; import org.apache.hadoop.fs.s3a.impl.CopyFromLocalOperation; import org.apache.hadoop.fs.s3a.impl.CreateFileBuilder; +import org.apache.hadoop.fs.s3a.impl.S3AFileSystemHandler; +import org.apache.hadoop.fs.s3a.impl.CSES3AFileSystemHandler; +import org.apache.hadoop.fs.s3a.impl.CSEV1CompatibleS3AFileSystemHandler; +import org.apache.hadoop.fs.s3a.impl.CSEMaterials; import org.apache.hadoop.fs.s3a.impl.DeleteOperation; import org.apache.hadoop.fs.s3a.impl.DirectoryPolicy; import org.apache.hadoop.fs.s3a.impl.DirectoryPolicyImpl; @@ -147,6 +151,7 @@ import org.apache.hadoop.fs.s3a.impl.StoreContextBuilder; import org.apache.hadoop.fs.s3a.impl.StoreContextFactory; import org.apache.hadoop.fs.s3a.impl.UploadContentProviders; +import org.apache.hadoop.fs.s3a.impl.CSEUtils; import org.apache.hadoop.fs.s3a.prefetch.S3APrefetchingInputStream; import org.apache.hadoop.fs.s3a.tools.MarkerToolOperations; import org.apache.hadoop.fs.s3a.tools.MarkerToolOperationsImpl; @@ -212,7 +217,6 @@ import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.util.Progressable; import org.apache.hadoop.util.RateLimitingFactory; -import org.apache.hadoop.util.ReflectionUtils; import org.apache.hadoop.util.SemaphoredDelegatingExecutor; import org.apache.hadoop.util.concurrent.HadoopExecutors; import org.apache.hadoop.util.functional.CallableRaisingIOE; @@ -249,7 +253,6 @@ import static org.apache.hadoop.fs.s3a.impl.HeaderProcessing.CONTENT_TYPE_OCTET_STREAM; import static org.apache.hadoop.fs.s3a.impl.InternalConstants.AP_REQUIRED_EXCEPTION; import static org.apache.hadoop.fs.s3a.impl.InternalConstants.ARN_BUCKET_OPTION; -import static org.apache.hadoop.fs.s3a.impl.InternalConstants.CSE_PADDING_LENGTH; import static org.apache.hadoop.fs.s3a.impl.InternalConstants.DEFAULT_UPLOAD_PART_COUNT_LIMIT; import static org.apache.hadoop.fs.s3a.impl.InternalConstants.SC_403_FORBIDDEN; import static org.apache.hadoop.fs.s3a.impl.InternalConstants.SC_404_NOT_FOUND; @@ -464,6 +467,12 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities, */ private ArnResource accessPoint; + /** + * Handler for certain filesystem operations. + */ + private S3AFileSystemHandler fsHandler; + + /** * Does this S3A FS instance have multipart upload enabled? */ @@ -622,11 +631,12 @@ public void initialize(URI name, Configuration originalConf) invoker = new Invoker(new S3ARetryPolicy(getConf()), onRetry); - // If CSE-KMS method is set then CSE is enabled. - isCSEEnabled = S3AEncryptionMethods.CSE_KMS.getMethod() - .equals(getS3EncryptionAlgorithm().getMethod()); - LOG.debug("Client Side Encryption enabled: {}", isCSEEnabled); - setCSEGauge(); + // If encryption method is set to CSE-KMS or CSE-CUSTOM then CSE is enabled. + isCSEEnabled = CSEUtils.isCSEEnabled(getS3EncryptionAlgorithm().getMethod()); + + // Create the appropriate fsHandler instance using a factory method + fsHandler = createFileSystemHandler(); + fsHandler.setCSEGauge((IOStatisticsStore) getIOStatistics()); // Username is the current user at the time the FS was instantiated. owner = UserGroupInformation.getCurrentUser(); username = owner.getShortUserName(); @@ -772,7 +782,7 @@ public void initialize(URI name, Configuration originalConf) BULK_DELETE_PAGE_SIZE_DEFAULT, 0); checkArgument(pageSize <= InternalConstants.MAX_ENTRIES_TO_DELETE, "page size out of range: %s", pageSize); - listing = new Listing(listingOperationCallbacks, createStoreContext()); + listing = new Listing(listingOperationCallbacks, createStoreContext(), fsHandler); // now the open file logic openFileHelper = new OpenFileSupport( changeDetectionPolicy, @@ -821,6 +831,26 @@ public void initialize(URI name, Configuration originalConf) } } + /** + * Creates and returns an instance of the appropriate S3AFileSystemHandler. + * Creation is baaed on the client-side encryption (CSE) settings. + * + * @return An instance of the appropriate S3AFileSystemHandler implementation. + */ + private S3AFileSystemHandler createFileSystemHandler() { + if (isCSEEnabled) { + if (getConf().getBoolean(S3_ENCRYPTION_CSE_V1_COMPATIBILITY_ENABLED, + S3_ENCRYPTION_CSE_V1_COMPATIBILITY_ENABLED_DEFAULT)) { + return new CSEV1CompatibleS3AFileSystemHandler(); + } else { + return new CSES3AFileSystemHandler(); + } + } else { + return new BaseS3AFileSystemHandler(); + } + } + + /** * Create the S3AStore instance. * This is protected so that tests can override it. @@ -862,22 +892,6 @@ private VectoredIOContext populateVectoredIOContext(Configuration conf) { .build(); } - /** - * Set the client side encryption gauge to 0 or 1, indicating if CSE is - * enabled through the gauge or not. - */ - private void setCSEGauge() { - IOStatisticsStore ioStatisticsStore = - (IOStatisticsStore) getIOStatistics(); - if (isCSEEnabled) { - ioStatisticsStore - .setGauge(CLIENT_SIDE_ENCRYPTION_ENABLED.getSymbol(), 1L); - } else { - ioStatisticsStore - .setGauge(CLIENT_SIDE_ENCRYPTION_ENABLED.getSymbol(), 0L); - } - } - /** * Test bucket existence in S3. * When the value of {@link Constants#S3A_BUCKET_PROBE} is set to 0, @@ -1123,9 +1137,11 @@ private ClientManager createClientManager(URI fsURI, boolean dtEnabled) throws I credentials = createAWSCredentialProviderList(fsURI, conf); } LOG.debug("Using credential provider {}", credentials); - Class s3ClientFactoryClass = conf.getClass( - S3_CLIENT_FACTORY_IMPL, DEFAULT_S3_CLIENT_FACTORY_IMPL, - S3ClientFactory.class); + + S3ClientFactory clientFactory = fsHandler.getS3ClientFactory(conf); + S3ClientFactory unencryptedClientFactory = fsHandler.getUnencryptedS3ClientFactory(conf); + CSEMaterials cseMaterials = fsHandler.getClientSideEncryptionMaterials(conf, bucket, + getS3EncryptionAlgorithm()); S3ClientFactory.S3ClientCreationParameters parameters = new S3ClientFactory.S3ClientCreationParameters() @@ -1146,17 +1162,20 @@ private ClientManager createClientManager(URI fsURI, boolean dtEnabled) throws I .withExpressCreateSession( conf.getBoolean(S3EXPRESS_CREATE_SESSION, S3EXPRESS_CREATE_SESSION_DEFAULT)) .withChecksumValidationEnabled( - conf.getBoolean(CHECKSUM_VALIDATION, CHECKSUM_VALIDATION_DEFAULT)); + conf.getBoolean(CHECKSUM_VALIDATION, CHECKSUM_VALIDATION_DEFAULT)) + .withClientSideEncryptionEnabled(isCSEEnabled) + .withClientSideEncryptionMaterials(cseMaterials); - S3ClientFactory clientFactory = ReflectionUtils.newInstance(s3ClientFactoryClass, conf); // this is where clients and the transfer manager are created on demand. - return createClientManager(clientFactory, parameters, getDurationTrackerFactory()); + return createClientManager(clientFactory, unencryptedClientFactory, parameters, + getDurationTrackerFactory()); } /** * Create the Client Manager; protected to allow for mocking. * Requires {@link #unboundedThreadPool} to be initialized. * @param clientFactory (reflection-bonded) client factory. + * @param unencryptedClientFactory (reflection-bonded) client factory. * @param clientCreationParameters parameters for client creation. * @param durationTrackerFactory factory for duration tracking. * @return a client manager instance. @@ -1164,9 +1183,11 @@ private ClientManager createClientManager(URI fsURI, boolean dtEnabled) throws I @VisibleForTesting protected ClientManager createClientManager( final S3ClientFactory clientFactory, + final S3ClientFactory unencryptedClientFactory, final S3ClientFactory.S3ClientCreationParameters clientCreationParameters, final DurationTrackerFactory durationTrackerFactory) { return new ClientManagerImpl(clientFactory, + unencryptedClientFactory, clientCreationParameters, durationTrackerFactory ); @@ -1942,10 +1963,11 @@ public GetObjectRequest.Builder newGetRequestBuilder(final String key) { } @Override - public ResponseInputStream getObject(GetObjectRequest request) { + public ResponseInputStream getObject(GetObjectRequest request) throws + IOException { // active the audit span used for the operation try (AuditSpan span = auditSpan.activate()) { - return getS3Client().getObject(request); + return fsHandler.getObject(store, request, getRequestFactory()); } } @@ -2624,14 +2646,8 @@ public RemoteIterator listFilesAndDirectoryMarkers( final S3AFileStatus status, final boolean includeSelf) throws IOException { auditSpan.activate(); - return innerListFiles( - path, - true, - includeSelf - ? Listing.ACCEPT_ALL_BUT_S3N - : new Listing.AcceptAllButSelfAndS3nDirs(path), - status - ); + return innerListFiles(path, true, + fsHandler.getFileStatusAcceptor(path, includeSelf), status); } @Override @@ -2677,7 +2693,7 @@ public RemoteIterator listObjects( listing.createFileStatusListingIterator(path, createListObjectsRequest(key, null), ACCEPT_ALL, - Listing.ACCEPT_ALL_BUT_S3N, + fsHandler.getFileStatusAcceptor(path, true), auditSpan)); } @@ -2773,6 +2789,19 @@ public long getDefaultBlockSize(Path path) { return S3AFileSystem.this.getDefaultBlockSize(path); } + /** + * Get the S3 object size. + * If the object is encrypted, the unpadded size will be returned. + * @param s3Object S3object + * @return plaintext S3 object size + * @throws IOException IO problems + */ + @Override + public long getObjectSize(S3Object s3Object) throws IOException { + return fsHandler.getS3ObjectSize(s3Object.key(), s3Object.size(), store, bucket, + getRequestFactory(), null); + } + @Override public int getMaxKeys() { return S3AFileSystem.this.getMaxKeys(); @@ -3051,8 +3080,14 @@ protected HeadObjectResponse getObjectMetadata(String key, if (changeTracker != null) { changeTracker.maybeApplyConstraint(requestBuilder); } - HeadObjectResponse headObjectResponse = getS3Client() - .headObject(requestBuilder.build()); + HeadObjectResponse headObjectResponse = getS3Client().headObject( + requestBuilder.build()); + long length = fsHandler.getS3ObjectSize(key, headObjectResponse.contentLength(), + store, bucket, getRequestFactory(), headObjectResponse); + // overwrite the content length + headObjectResponse = headObjectResponse.toBuilder() + .contentLength(length) + .build(); if (changeTracker != null) { changeTracker.processMetadata(headObjectResponse, operation); } @@ -3727,7 +3762,7 @@ private RemoteIterator innerListStatus(Path f) return listing.createProvidedFileStatusIterator( stats, ACCEPT_ALL, - Listing.ACCEPT_ALL_BUT_S3N); + fsHandler.getFileStatusAcceptor(path, true)); } } // Here we have a directory which may or may not be empty. @@ -3925,7 +3960,8 @@ public S3AFileStatus probePathStatus(final Path path, @Override public RemoteIterator listFilesIterator(final Path path, final boolean recursive) throws IOException { - return S3AFileSystem.this.innerListFiles(path, recursive, Listing.ACCEPT_ALL_BUT_S3N, null); + return S3AFileSystem.this.innerListFiles(path, recursive, + fsHandler.getFileStatusAcceptor(path, true), null); } } @@ -4084,14 +4120,7 @@ S3AFileStatus s3GetFileStatus(final Path path, // look for the simple file HeadObjectResponse meta = getObjectMetadata(key); LOG.debug("Found exact file: normal file {}", key); - long contentLength = meta.contentLength(); - // check if CSE is enabled, then strip padded length. - if (isCSEEnabled && - meta.metadata().get(AWSHeaders.CRYPTO_CEK_ALGORITHM) != null - && contentLength >= CSE_PADDING_LENGTH) { - contentLength -= CSE_PADDING_LENGTH; - } - return new S3AFileStatus(contentLength, + return new S3AFileStatus(meta.contentLength(), meta.lastModified().toEpochMilli(), path, getDefaultBlockSize(path), @@ -5224,7 +5253,7 @@ public RemoteIterator listFiles(Path f, return toLocatedFileStatusIterator( trackDurationAndSpan(INVOCATION_LIST_FILES, path, () -> innerListFiles(path, recursive, - new Listing.AcceptFilesOnly(path), null))); + fsHandler.getFileStatusAcceptor(path), null))); } /** @@ -5242,7 +5271,7 @@ public RemoteIterator listFilesAndEmptyDirectories( final Path path = qualify(f); return trackDurationAndSpan(INVOCATION_LIST_FILES, path, () -> innerListFiles(path, recursive, - Listing.ACCEPT_ALL_BUT_S3N, + fsHandler.getFileStatusAcceptor(path, true), null)); } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInputStream.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInputStream.java index cfdc361234f9f9..147cd7567a301f 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInputStream.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInputStream.java @@ -1405,11 +1405,15 @@ public interface InputStreamCallbacks extends Closeable { /** * Execute the request. + * When CSE is enabled with reading of unencrypted data, The object is checked if it is + * encrypted and if so, the request is made with encrypted S3 client. If the object is + * not encrypted, the request is made with unencrypted s3 client. * @param request the request * @return the response + * @throws IOException on any failure. */ @Retries.OnceRaw - ResponseInputStream getObject(GetObjectRequest request); + ResponseInputStream getObject(GetObjectRequest request) throws IOException; /** * Submit some asynchronous work, for example, draining a stream. diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java index 69390a8cc724e8..741607c7b8ad37 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java @@ -529,7 +529,7 @@ public static String stringify(AwsServiceException e) { * @param owner owner of the file * @param eTag S3 object eTag or null if unavailable * @param versionId S3 object versionId or null if unavailable - * @param isCSEEnabled is client side encryption enabled? + * @param size s3 object size * @return a status entry */ public static S3AFileStatus createFileStatus(Path keyPath, @@ -538,12 +538,7 @@ public static S3AFileStatus createFileStatus(Path keyPath, String owner, String eTag, String versionId, - boolean isCSEEnabled) { - long size = s3Object.size(); - // check if cse is enabled; strip out constant padding length. - if (isCSEEnabled && size >= CSE_PADDING_LENGTH) { - size -= CSE_PADDING_LENGTH; - } + long size) { return createFileStatus(keyPath, objectRepresentsDirectory(s3Object.key()), size, Date.from(s3Object.lastModified()), blockSize, owner, eTag, versionId); diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java index e82eb4c9182e16..c18b7a1dcdd8f6 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java @@ -34,6 +34,7 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.fs.s3a.impl.CSEMaterials; import org.apache.hadoop.fs.s3a.statistics.StatisticsFromAwsSdk; import static org.apache.hadoop.fs.s3a.Constants.DEFAULT_ENDPOINT; @@ -118,6 +119,17 @@ final class S3ClientCreationParameters { */ private StatisticsFromAwsSdk metrics; + /** + * Is CSE enabled? + * The default value is {@value}. + */ + private Boolean isCSEEnabled = false; + + /** + * Client side encryption materials. + */ + private CSEMaterials cseMaterials; + /** * Use (deprecated) path style access. */ @@ -428,6 +440,44 @@ public S3ClientCreationParameters withRegion( return this; } + /** + * Set the client side encryption flag. + * + * @param value new value + * @return the builder + */ + public S3ClientCreationParameters withClientSideEncryptionEnabled(final boolean value) { + this.isCSEEnabled = value; + return this; + } + + /** + * Get the client side encryption flag. + * @return client side encryption flag + */ + public boolean isClientSideEncryptionEnabled() { + return this.isCSEEnabled; + } + + /** + * Set the client side encryption materials. + * + * @param value new value + * @return the builder + */ + public S3ClientCreationParameters withClientSideEncryptionMaterials(final CSEMaterials value) { + this.cseMaterials = value; + return this; + } + + /** + * Get the client side encryption materials. + * @return client side encryption materials + */ + public CSEMaterials getClientSideEncryptionMaterials() { + return this.cseMaterials; + } + /** * Get the region. * @return invoker diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/WriteOperationHelper.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/WriteOperationHelper.java index b7387fc12e1402..a7a87bdfcfb299 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/WriteOperationHelper.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/WriteOperationHelper.java @@ -469,6 +469,7 @@ public void abortMultipartCommit(String destKey, String uploadId) * @param destKey destination key of ongoing operation * @param uploadId ID of ongoing upload * @param partNumber current part number of the upload + * @param isLastPart is this the last part? * @param size amount of data * @return the request builder. * @throws IllegalArgumentException if the parameters are invalid. @@ -480,6 +481,7 @@ public UploadPartRequest.Builder newUploadPartRequestBuilder( String destKey, String uploadId, int partNumber, + boolean isLastPart, long size) throws IOException { return once("upload part request", destKey, withinAuditSpan(getAuditSpan(), () -> @@ -487,6 +489,7 @@ public UploadPartRequest.Builder newUploadPartRequestBuilder( destKey, uploadId, partNumber, + isLastPart, size))); } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/WriteOperations.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/WriteOperations.java index 68709c40f45ca3..93d2506a4f3ad1 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/WriteOperations.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/WriteOperations.java @@ -190,6 +190,7 @@ void abortMultipartCommit(String destKey, String uploadId) * @param destKey destination key of ongoing operation * @param uploadId ID of ongoing upload * @param partNumber current part number of the upload + * @param isLastPart is this the last part of the upload? * @param size amount of data * @return the request builder. * @throws IllegalArgumentException if the parameters are invalid @@ -199,6 +200,7 @@ UploadPartRequest.Builder newUploadPartRequestBuilder( String destKey, String uploadId, int partNumber, + boolean isLastPart, long size) throws IOException; /** diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/api/RequestFactory.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/api/RequestFactory.java index 73ad137a86d3ca..c69e3394c3dd33 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/api/RequestFactory.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/api/RequestFactory.java @@ -203,6 +203,7 @@ CompleteMultipartUploadRequest.Builder newCompleteMultipartUploadRequestBuilder( * @param destKey destination key of ongoing operation * @param uploadId ID of ongoing upload * @param partNumber current part number of the upload + * @param isLastPart isLastPart is this the last part? * @param size amount of data * @return the request builder. * @throws PathIOException if the part number is out of range. @@ -211,6 +212,7 @@ UploadPartRequest.Builder newUploadPartRequestBuilder( String destKey, String uploadId, int partNumber, + boolean isLastPart, long size) throws PathIOException; /** diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/impl/CommitOperations.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/impl/CommitOperations.java index f33d94ce84fef6..14bd4cc2f7da1a 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/impl/CommitOperations.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/impl/CommitOperations.java @@ -638,17 +638,19 @@ private List uploadFileData( for (int partNumber = 1; partNumber <= numParts; partNumber++) { progress.progress(); int size = (int)Math.min(length - offset, uploadPartSize); - UploadPartRequest part = writeOperations.newUploadPartRequestBuilder( + UploadPartRequest.Builder partBuilder = writeOperations.newUploadPartRequestBuilder( destKey, uploadId, partNumber, - size).build(); + partNumber == numParts, + size); // Create a file content provider starting at the current offset. RequestBody body = RequestBody.fromContentProvider( UploadContentProviders.fileContentProvider(localFile, offset, size), size, CONTENT_TYPE_OCTET_STREAM); - UploadPartResponse response = writeOperations.uploadPart(part, body, statistics); + UploadPartResponse response = writeOperations.uploadPart(partBuilder.build(), body, + statistics); offset += uploadPartSize; parts.add(CompletedPart.builder() .partNumber(partNumber) diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/AWSHeaders.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/AWSHeaders.java index aaca3b9b194d62..fe7b45147986f9 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/AWSHeaders.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/AWSHeaders.java @@ -83,6 +83,11 @@ public interface AWSHeaders { */ String CRYPTO_CEK_ALGORITHM = "x-amz-cek-alg"; + /** + * Header for unencrypted content length of an object: {@value}. + */ + String UNENCRYPTED_CONTENT_LENGTH = "x-amz-unencrypted-content-length"; + /** * Headers in request indicating that the requester must be charged for data * transfer. diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/BaseS3AFileSystemHandler.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/BaseS3AFileSystemHandler.java new file mode 100644 index 00000000000000..a0b3e75d55af17 --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/BaseS3AFileSystemHandler.java @@ -0,0 +1,163 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.s3a.impl; + +import java.io.IOException; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.s3a.Listing; +import org.apache.hadoop.fs.s3a.S3AEncryptionMethods; +import org.apache.hadoop.fs.s3a.S3AStore; +import org.apache.hadoop.fs.s3a.S3ClientFactory; +import org.apache.hadoop.fs.s3a.api.RequestFactory; +import org.apache.hadoop.fs.statistics.impl.IOStatisticsStore; +import org.apache.hadoop.util.ReflectionUtils; + +import software.amazon.awssdk.core.ResponseInputStream; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; +import software.amazon.awssdk.services.s3.model.GetObjectRequest; +import software.amazon.awssdk.services.s3.model.HeadObjectResponse; + +import static org.apache.hadoop.fs.s3a.Constants.DEFAULT_S3_CLIENT_FACTORY_IMPL; +import static org.apache.hadoop.fs.s3a.Constants.S3_CLIENT_FACTORY_IMPL; +import static org.apache.hadoop.fs.s3a.Statistic.CLIENT_SIDE_ENCRYPTION_ENABLED; + +/** + * An implementation of the {@link S3AFileSystemHandler} interface. + * This handles certain filesystem operations when s3 client side encryption is disabled. + */ +public class BaseS3AFileSystemHandler implements S3AFileSystemHandler { + + /** + * Constructs a new instance of {@code BaseS3AFileSystemHandler}. + */ + public BaseS3AFileSystemHandler() { + } + + /** + * Returns a {@link Listing.FileStatusAcceptor} object. + * That determines which files and directories should be included in a listing operation. + * + * @param path the path for which the listing is being performed + * @param includeSelf a boolean indicating whether the path itself should + * be included in the listing + * @return a {@link Listing.FileStatusAcceptor} object + */ + @Override + public Listing.FileStatusAcceptor getFileStatusAcceptor(Path path, boolean includeSelf) { + return includeSelf + ? Listing.ACCEPT_ALL_BUT_S3N + : new Listing.AcceptAllButSelfAndS3nDirs(path); + } + + /** + * Returns a {@link Listing.FileStatusAcceptor} object. + * That determines which files and directories should be included in a listing operation. + * + * @param path the path for which the listing is being performed + * @return a {@link Listing.FileStatusAcceptor} object + */ + @Override + public Listing.FileStatusAcceptor getFileStatusAcceptor(Path path) { + return new Listing.AcceptFilesOnly(path); + } + + /** + * Retrieves an object from the S3. + * + * @param store The S3AStore object representing the S3 bucket. + * @param request The GetObjectRequest containing the details of the object to retrieve. + * @param factory The RequestFactory used to create the GetObjectRequest. + * @return A ResponseInputStream containing the GetObjectResponse. + * @throws IOException If an error occurs while retrieving the object. + */ + @Override + public ResponseInputStream getObject(S3AStore store, GetObjectRequest request, + RequestFactory factory) throws IOException { + return store.getOrCreateS3Client().getObject(request); + } + + /** + * Set the client side encryption gauge to 0. + * @param ioStatisticsStore The IOStatisticsStore of the filesystem. + */ + @Override + public void setCSEGauge(IOStatisticsStore ioStatisticsStore) { + ioStatisticsStore.setGauge(CLIENT_SIDE_ENCRYPTION_ENABLED.getSymbol(), 0L); + } + + /** + * Retrieves the client-side encryption materials for the given bucket and encryption algorithm. + * + * @param conf The Hadoop configuration object. + * @param bucket The name of the S3 bucket. + * @param algorithm The client-side encryption algorithm to use. + * @return null. + */ + @Override + public CSEMaterials getClientSideEncryptionMaterials(Configuration conf, String bucket, + S3AEncryptionMethods algorithm) { + return null; + } + + /** + * Retrieves the S3 client factory for the specified class and configuration. + * + * @param conf The Hadoop configuration object. + * @return The S3 client factory instance. + */ + @Override + public S3ClientFactory getS3ClientFactory(Configuration conf) { + Class s3ClientFactoryClass = conf.getClass( + S3_CLIENT_FACTORY_IMPL, DEFAULT_S3_CLIENT_FACTORY_IMPL, + S3ClientFactory.class); + return ReflectionUtils.newInstance(s3ClientFactoryClass, conf); + } + + /** + * Retrieves the S3 client factory for the specified class and configuration. + * + * @param conf The Hadoop configuration object. + * @return null. + */ + @Override + public S3ClientFactory getUnencryptedS3ClientFactory(Configuration conf) { + return null; + } + + + /** + * Return the size of S3 object. + * + * @param key The key (path) of the object in the S3 bucket. + * @param length The expected length of the object. + * @param store The S3AStore object representing the S3 bucket. + * @param bucket The name of the S3 bucket. + * @param factory The RequestFactory used to create the HeadObjectRequest. + * @param response The HeadObjectResponse containing the metadata of the object. + * @return The size of the object in bytes. + */ + @Override + public long getS3ObjectSize(String key, long length, S3AStore store, String bucket, + RequestFactory factory, HeadObjectResponse response) { + return length; + } + +} diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/CSEMaterials.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/CSEMaterials.java new file mode 100644 index 00000000000000..adbeb441fbca45 --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/CSEMaterials.java @@ -0,0 +1,132 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.s3a.impl; + +import org.apache.hadoop.conf.Configuration; + +/** + * This class is for storing information about key type and corresponding key + * to be used for client side encryption. + */ +public class CSEMaterials { + + /** + * Enum for CSE key types. + */ + public enum CSEKeyType { + /** + * AWS KMS keys are used of encryption and decryption. + */ + KMS, + /** + * Custom cryptographic manager class is used for encryption and decryption. + */ + CUSTOM + } + + /** + * The KMS key Id. + */ + private String kmsKeyId; + + /** + * Custom cryptographic manager class name. + */ + private String customKeyringClassName; + + private Configuration conf; + + /** + * The CSE key type to use. + */ + private CSEKeyType cseKeyType; + + /** + * Kms key id to use. + * @param value new value + * @return the builder + */ + public CSEMaterials withKmsKeyId( + final String value) { + kmsKeyId = value; + return this; + } + + /** + * Custom cryptographic class name to use. + * @param value cryptographic manager class name + * @return the builder + */ + public CSEMaterials withCustomCryptographicClassName( + final String value) { + customKeyringClassName = value; + return this; + } + + /** + * Configuration. + * @param value configuration + * @return the builder + */ + public CSEMaterials withConf( + final Configuration value) { + conf = value; + return this; + } + + + /** + * Get the Kms key id to use. + * @return the kms key id. + */ + public String getKmsKeyId() { + return kmsKeyId; + } + + public Configuration getConf() { + return conf; + } + + /** + * Get the custom cryptographic class name. + * @return custom keyring class name + */ + public String getCustomKeyringClassName() { + return customKeyringClassName; + } + + /** + * CSE key type to use. + * @param value new value + * @return the builder + */ + public CSEMaterials withCSEKeyType( + final CSEKeyType value) { + cseKeyType = value; + return this; + } + + /** + * Get the CSE key type. + * @return CSE key type + */ + public CSEKeyType getCseKeyType() { + return cseKeyType; + } +} diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/CSES3AFileSystemHandler.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/CSES3AFileSystemHandler.java new file mode 100644 index 00000000000000..66a76c1028d23b --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/CSES3AFileSystemHandler.java @@ -0,0 +1,166 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.s3a.impl; + +import java.io.IOException; + +import software.amazon.awssdk.core.ResponseInputStream; +import software.amazon.awssdk.services.s3.model.GetObjectRequest; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; +import software.amazon.awssdk.services.s3.model.HeadObjectResponse; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.s3a.Listing; +import org.apache.hadoop.fs.s3a.S3AEncryptionMethods; +import org.apache.hadoop.fs.s3a.S3AStore; +import org.apache.hadoop.fs.s3a.S3ClientFactory; +import org.apache.hadoop.fs.s3a.api.RequestFactory; +import org.apache.hadoop.fs.statistics.impl.IOStatisticsStore; +import org.apache.hadoop.util.ReflectionUtils; + +import static org.apache.hadoop.fs.s3a.Statistic.CLIENT_SIDE_ENCRYPTION_ENABLED; +import static org.apache.hadoop.fs.s3a.impl.InternalConstants.CSE_PADDING_LENGTH; + +/** + * An implementation of the {@link S3AFileSystemHandler} interface. + * This handles certain filesystem operations when s3 client side encryption is enabled. + */ +public class CSES3AFileSystemHandler implements S3AFileSystemHandler{ + + /** + * Constructs a new instance of {@code CSES3AFileSystemHandler}. + */ + public CSES3AFileSystemHandler() { + } + + /** + * Returns a {@link Listing.FileStatusAcceptor} object. + * That determines which files and directories should be included in a listing operation. + * + * @param path the path for which the listing is being performed + * @param includeSelf a boolean indicating whether the path itself should + * be included in the listing + * @return a {@link Listing.FileStatusAcceptor} object + */ + @Override + public Listing.FileStatusAcceptor getFileStatusAcceptor(Path path, boolean includeSelf) { + return includeSelf + ? Listing.ACCEPT_ALL_BUT_S3N + : new Listing.AcceptAllButSelfAndS3nDirs(path); + } + + /** + * Returns a {@link Listing.FileStatusAcceptor} object. + * That determines which files and directories should be included in a listing operation. + * + * @param path the path for which the listing is being performed + * @return a {@link Listing.FileStatusAcceptor} object + */ + @Override + public Listing.FileStatusAcceptor getFileStatusAcceptor(Path path) { + return new Listing.AcceptFilesOnly(path); + } + + /** + * Retrieves an object from the S3 using encrypted S3 client. + * + * @param store The S3AStore object representing the S3 bucket. + * @param request The GetObjectRequest containing the details of the object to retrieve. + * @param factory The RequestFactory used to create the GetObjectRequest. + * @return A ResponseInputStream containing the GetObjectResponse. + * @throws IOException If an error occurs while retrieving the object. + */ + @Override + public ResponseInputStream getObject(S3AStore store, GetObjectRequest request, + RequestFactory factory) throws IOException { + return store.getOrCreateS3Client().getObject(request); + } + + /** + * Set the client side encryption gauge 1. + * @param ioStatisticsStore The IOStatisticsStore of the filesystem. + */ + @Override + public void setCSEGauge(IOStatisticsStore ioStatisticsStore) { + ioStatisticsStore.setGauge(CLIENT_SIDE_ENCRYPTION_ENABLED.getSymbol(), 1L); + } + + /** + * Retrieves the client-side encryption materials for the given bucket and encryption algorithm. + * + * @param conf The Hadoop configuration object. + * @param bucket The name of the S3 bucket. + * @param algorithm The client-side encryption algorithm to use. + * @return The client-side encryption materials. + * @throws IOException If an error occurs while retrieving the encryption materials. + */ + @Override + public CSEMaterials getClientSideEncryptionMaterials(Configuration conf, String bucket, + S3AEncryptionMethods algorithm) throws IOException { + return CSEUtils.getClientSideEncryptionMaterials(conf, bucket, algorithm); + } + + /** + * Retrieves the S3 client factory for the specified class and configuration. + * + * @param conf The Hadoop configuration object. + * @return The S3 client factory instance. + */ + @Override + public S3ClientFactory getS3ClientFactory(Configuration conf) { + return ReflectionUtils.newInstance(EncryptionS3ClientFactory.class, conf); + } + + /** + * Retrieves the S3 client factory for the specified class and configuration. + * + * @param conf The Hadoop configuration object. + * @return null + */ + @Override + public S3ClientFactory getUnencryptedS3ClientFactory(Configuration conf) { + return null; + } + + + /** + * Retrieves the unpadded size of an object in the S3. + * + * @param key The key (path) of the object in the S3 bucket. + * @param length The expected length of the object, including any padding. + * @param store The S3AStore object representing the S3 bucket. + * @param bucket The name of the S3 bucket. + * @param factory The RequestFactory used to create the HeadObjectRequest. + * @param response The HeadObjectResponse containing the metadata of the object. + * @return The unpadded size of the object in bytes. + * @throws IOException If an error occurs while retrieving the object size. + */ + @Override + public long getS3ObjectSize(String key, long length, S3AStore store, String bucket, + RequestFactory factory, HeadObjectResponse response) throws IOException { + long unpaddedLength = length - CSE_PADDING_LENGTH; + if (unpaddedLength >= 0) { + return unpaddedLength; + } + return length; + } + + +} diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/CSEUtils.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/CSEUtils.java new file mode 100644 index 00000000000000..100fdc0ec54380 --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/CSEUtils.java @@ -0,0 +1,196 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.s3a.impl; + +import java.io.IOException; +import java.io.InputStream; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.s3a.S3AEncryptionMethods; +import org.apache.hadoop.fs.s3a.api.RequestFactory; +import org.apache.hadoop.util.Preconditions; + +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.GetObjectRequest; +import software.amazon.awssdk.services.s3.model.HeadObjectRequest; +import software.amazon.awssdk.services.s3.model.HeadObjectResponse; +import software.amazon.awssdk.services.s3.model.NoSuchKeyException; + +import static org.apache.hadoop.fs.s3a.Constants.S3_ENCRYPTION_CSE_CUSTOM_KEYRING_CLASS_NAME; +import static org.apache.hadoop.fs.s3a.Constants.S3_ENCRYPTION_CSE_INSTRUCTION_FILE_SUFFIX; +import static org.apache.hadoop.fs.s3a.S3AEncryptionMethods.CSE_CUSTOM; +import static org.apache.hadoop.fs.s3a.S3AEncryptionMethods.CSE_KMS; +import static org.apache.hadoop.fs.s3a.S3AUtils.formatRange; +import static org.apache.hadoop.fs.s3a.S3AUtils.getS3EncryptionKey; +import static org.apache.hadoop.fs.s3a.impl.AWSHeaders.CRYPTO_CEK_ALGORITHM; +import static org.apache.hadoop.fs.s3a.impl.AWSHeaders.UNENCRYPTED_CONTENT_LENGTH; +import static org.apache.hadoop.fs.s3a.impl.InternalConstants.CSE_PADDING_LENGTH; + +/** + * S3 client side encryption (CSE) utility class. + */ +@InterfaceAudience.Public +@InterfaceStability.Evolving +public final class CSEUtils { + + private CSEUtils() { + } + + /** + * Checks if the file suffix ends CSE file suffix. + * {@link org.apache.hadoop.fs.s3a.Constants#S3_ENCRYPTION_CSE_INSTRUCTION_FILE_SUFFIX} + * when the config + * @param key file name + * @return true if file name ends with CSE instruction file suffix + */ + public static boolean isCSEInstructionFile(String key) { + return key.endsWith(S3_ENCRYPTION_CSE_INSTRUCTION_FILE_SUFFIX); + } + + /** + * Checks if CSE-KMS or CSE-CUSTOM is set. + * @param encryptionMethod type of encryption used + * @return true if encryption method is CSE-KMS or CSE-CUSTOM + */ + public static boolean isCSEEnabled(String encryptionMethod) { + return CSE_KMS.getMethod().equals(encryptionMethod) || + CSE_CUSTOM.getMethod().equals(encryptionMethod); + } + + /** + * Checks if a given S3 object is encrypted or not by checking following two conditions + * 1. if object metadata contains x-amz-cek-alg + * 2. if instruction file is present + * + * @param s3Client S3 client + * @param factory S3 request factory + * @param key key value of the s3 object + * @return true if S3 object is encrypted + */ + public static boolean isObjectEncrypted(S3Client s3Client, RequestFactory factory, String key) { + HeadObjectRequest.Builder requestBuilder = factory.newHeadObjectRequestBuilder(key); + HeadObjectResponse headObjectResponse = s3Client.headObject(requestBuilder.build()); + if (headObjectResponse.hasMetadata() && + headObjectResponse.metadata().get(CRYPTO_CEK_ALGORITHM) != null) { + return true; + } + HeadObjectRequest.Builder instructionFileRequestBuilder = + factory.newHeadObjectRequestBuilder(key + S3_ENCRYPTION_CSE_INSTRUCTION_FILE_SUFFIX); + try { + s3Client.headObject(instructionFileRequestBuilder.build()); + return true; + } catch (NoSuchKeyException e) { + // Ignore. This indicates no instruction file is present + } + return false; + } + + /** + * Get the unencrypted length of the object. + * + * @param s3Client S3 client + * @param bucket bucket name of the s3 object + * @param key key value of the s3 object + * @param factory S3 request factory + * @param contentLength S3 object length + * @param headObjectResponse response from headObject call + * @return unencrypted length of the object + * @throws IOException IO failures + */ + public static long getUnPaddedObjectLength(S3Client s3Client, + String bucket, + String key, + RequestFactory factory, + long contentLength, + HeadObjectResponse headObjectResponse) throws IOException { + + // if object is unencrypted, return the actual size + if (!isObjectEncrypted(s3Client, factory, key)) { + return contentLength; + } + + // check if unencrypted content length metadata is present or not. + if (headObjectResponse != null) { + String plaintextLength = headObjectResponse.metadata().get(UNENCRYPTED_CONTENT_LENGTH); + if (headObjectResponse.hasMetadata() && plaintextLength != null + && !plaintextLength.isEmpty()) { + return Long.parseLong(plaintextLength); + } + } + + // identify the length by doing a ranged GET operation. + if (contentLength >= CSE_PADDING_LENGTH) { + long minPlaintextLength = contentLength - CSE_PADDING_LENGTH; + if (minPlaintextLength < 0) { + minPlaintextLength = 0; + } + GetObjectRequest getObjectRequest = GetObjectRequest.builder() + .bucket(bucket) + .key(key) + .range(formatRange(minPlaintextLength, contentLength)) + .build(); + try (InputStream is = s3Client.getObject(getObjectRequest)) { + int i = 0; + while (is.read() != -1) { + i++; + } + return minPlaintextLength + i; + } catch (Exception e) { + throw new IOException("Failed to compute unencrypted length", e); + } + } + return contentLength; + } + + /** + * Configure CSE params based on encryption algorithm. + * @param conf Configuration + * @param bucket bucket name + * @param algorithm encryption algorithm + * @return CSEMaterials + * @throws IOException IO failures + */ + public static CSEMaterials getClientSideEncryptionMaterials(Configuration conf, String bucket, + S3AEncryptionMethods algorithm) throws IOException { + switch (algorithm) { + case CSE_KMS: + String kmsKeyId = getS3EncryptionKey(bucket, conf, true); + Preconditions.checkArgument(kmsKeyId != null && !kmsKeyId.isEmpty(), + "KMS keyId cannot be null or empty"); + return new CSEMaterials() + .withCSEKeyType(CSEMaterials.CSEKeyType.KMS) + .withConf(conf) + .withKmsKeyId(kmsKeyId); + case CSE_CUSTOM: + String customCryptoClassName = conf.getTrimmed(S3_ENCRYPTION_CSE_CUSTOM_KEYRING_CLASS_NAME); + Preconditions.checkArgument(customCryptoClassName != null && + !customCryptoClassName.isEmpty(), + "CSE custom cryptographic class name cannot be null or empty"); + return new CSEMaterials() + .withCSEKeyType(CSEMaterials.CSEKeyType.CUSTOM) + .withConf(conf) + .withCustomCryptographicClassName(customCryptoClassName); + default: + throw new IllegalArgumentException("Invalid client side encryption algorithm." + + " Only CSE-KMS and CSE-CUSTOM are supported"); + } + } +} diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/CSEV1CompatibleS3AFileSystemHandler.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/CSEV1CompatibleS3AFileSystemHandler.java new file mode 100644 index 00000000000000..25b09a4680d056 --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/CSEV1CompatibleS3AFileSystemHandler.java @@ -0,0 +1,132 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.s3a.impl; + +import java.io.IOException; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.s3a.Listing; +import org.apache.hadoop.fs.s3a.S3AStore; +import org.apache.hadoop.fs.s3a.S3ClientFactory; +import org.apache.hadoop.fs.s3a.api.RequestFactory; +import org.apache.hadoop.util.ReflectionUtils; + +import software.amazon.awssdk.core.ResponseInputStream; +import software.amazon.awssdk.services.s3.model.GetObjectRequest; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; +import software.amazon.awssdk.services.s3.model.HeadObjectResponse; + +import static org.apache.hadoop.fs.s3a.Constants.DEFAULT_S3_CLIENT_FACTORY_IMPL; +import static org.apache.hadoop.fs.s3a.Constants.S3_CLIENT_FACTORY_IMPL; +import static org.apache.hadoop.fs.s3a.impl.CSEUtils.isObjectEncrypted; + +/** + * An extension of the {@link CSES3AFileSystemHandler} class. + * This handles certain file system operations when client-side encryption is enabled with v1 client + * compatibility. + * {@link org.apache.hadoop.fs.s3a.Constants#S3_ENCRYPTION_CSE_V1_COMPATIBILITY_ENABLED}. + */ +public class CSEV1CompatibleS3AFileSystemHandler extends CSES3AFileSystemHandler { + + /** + * Constructs a new instance of {@code CSEV1CompatibleS3AFileSystemHandler}. + */ + public CSEV1CompatibleS3AFileSystemHandler() { + } + + /** + * {@inheritDoc} + * + *

This implementation returns a {@link Listing.AcceptAllButS3nDirsAndCSEInstructionFile} + * or {@link Listing.AcceptAllButSelfAndS3nDirsAndCSEInstructionFile} object + * based on the value of the {@code includeSelf} parameter. + */ + @Override + public Listing.FileStatusAcceptor getFileStatusAcceptor(Path path, boolean includeSelf) { + return includeSelf + ? new Listing.AcceptAllButS3nDirsAndCSEInstructionFile() + : new Listing.AcceptAllButSelfAndS3nDirsAndCSEInstructionFile(path); + } + + /** + * Returns a {@link Listing.FileStatusAcceptor} object. + * That determines which files and directories should be included in a listing operation. + * + * @param path the path for which the listing is being performed + * @return a {@link Listing.FileStatusAcceptor} object + */ + @Override + public Listing.FileStatusAcceptor getFileStatusAcceptor(Path path) { + return new Listing.AcceptFilesOnlyExceptCSEInstructionFile(path); + } + + /** + * Retrieves an object from the S3. + * If the S3 object is encrypted, it uses the encrypted S3 client to retrieve the object else + * it uses the unencrypted S3 client. + * + * @param store The S3AStore object representing the S3 bucket. + * @param request The GetObjectRequest containing the details of the object to retrieve. + * @param factory The RequestFactory used to create the GetObjectRequest. + * @return A ResponseInputStream containing the GetObjectResponse. + * @throws IOException If an error occurs while retrieving the object. + */ + @Override + public ResponseInputStream getObject(S3AStore store, GetObjectRequest request, + RequestFactory factory) throws IOException { + boolean isEncrypted = isObjectEncrypted(store.getOrCreateS3Client(), factory, request.key()); + return isEncrypted ? store.getOrCreateS3Client().getObject(request) + : store.getOrCreateUnencryptedS3Client().getObject(request); + } + + /** + * Retrieves the S3 client factory for the specified class and configuration. + * + * @param conf The Hadoop configuration object. + * @return The S3 client factory instance. + */ + @Override + public S3ClientFactory getUnencryptedS3ClientFactory(Configuration conf) { + Class s3ClientFactoryClass = conf.getClass( + S3_CLIENT_FACTORY_IMPL, DEFAULT_S3_CLIENT_FACTORY_IMPL, + S3ClientFactory.class); + return ReflectionUtils.newInstance(s3ClientFactoryClass, conf); + } + + + /** + * Retrieves the unpadded size of an object in the S3 bucket. + * + * @param key The key (path) of the object in the S3 bucket. + * @param length The length of the object. + * @param store The S3AStore object representing the S3 bucket. + * @param bucket The name of the S3 bucket. + * @param factory The RequestFactory used to create the HeadObjectRequest. + * @param response The HeadObjectResponse containing the metadata of the object. + * @return The unpadded size of the object in bytes. + * @throws IOException If an error occurs while retrieving the object size. + */ + @Override + public long getS3ObjectSize(String key, long length, S3AStore store, String bucket, + RequestFactory factory, HeadObjectResponse response) throws IOException { + return CSEUtils.getUnPaddedObjectLength(store.getOrCreateS3Client(), bucket, + key, factory, length, response); + } +} diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/ClientManager.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/ClientManager.java index 7fadac8623d50d..c9eaeebae4b1b3 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/ClientManager.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/ClientManager.java @@ -61,6 +61,8 @@ S3TransferManager getOrCreateTransferManager() */ S3AsyncClient getOrCreateAsyncClient() throws IOException; + S3Client getOrCreateUnencryptedS3Client() throws IOException; + /** * Get the AsyncS3Client, raising a failure to create as an UncheckedIOException. * @return the S3 client diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/ClientManagerImpl.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/ClientManagerImpl.java index 24c37cc564a095..44383e381248fb 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/ClientManagerImpl.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/ClientManagerImpl.java @@ -62,6 +62,11 @@ public class ClientManagerImpl implements ClientManager { */ private final S3ClientFactory clientFactory; + /** + * Client factory to invoke for unencrypted client. + */ + private final S3ClientFactory unencryptedClientFactory; + /** * Closed flag. */ @@ -85,6 +90,12 @@ public class ClientManagerImpl implements ClientManager { /** Async client is used for transfer manager. */ private final LazyAutoCloseableReference s3AsyncClient; + /** + * Unencrypted S3 client. + * This is used for unencrypted operations when CSE is enabled with V1 compatibility. + */ + private final LazyAutoCloseableReference unencryptedS3Client; + /** Transfer manager. */ private final LazyAutoCloseableReference transferManager; @@ -95,18 +106,22 @@ public class ClientManagerImpl implements ClientManager { *

* It does disable noisy logging from the S3 Transfer Manager. * @param clientFactory client factory to invoke + * @param unencryptedClientFactory client factory to invoke * @param clientCreationParameters creation parameters. * @param durationTrackerFactory duration tracker. */ public ClientManagerImpl( final S3ClientFactory clientFactory, + final S3ClientFactory unencryptedClientFactory, final S3ClientFactory.S3ClientCreationParameters clientCreationParameters, final DurationTrackerFactory durationTrackerFactory) { this.clientFactory = requireNonNull(clientFactory); + this.unencryptedClientFactory = unencryptedClientFactory; this.clientCreationParameters = requireNonNull(clientCreationParameters); this.durationTrackerFactory = requireNonNull(durationTrackerFactory); this.s3Client = new LazyAutoCloseableReference<>(createS3Client()); this.s3AsyncClient = new LazyAutoCloseableReference<>(createAyncClient()); + this.unencryptedS3Client = new LazyAutoCloseableReference<>(createUnencryptedS3Client()); this.transferManager = new LazyAutoCloseableReference<>(createTransferManager()); // fix up SDK logging. @@ -135,6 +150,17 @@ private CallableRaisingIOE createAyncClient() { () -> clientFactory.createS3AsyncClient(getUri(), clientCreationParameters)); } + /** + * Create the function to create the unencrypted S3 client. + * @return a callable which will create the client. + */ + private CallableRaisingIOE createUnencryptedS3Client() { + return trackDurationOfOperation( + durationTrackerFactory, + STORE_CLIENT_CREATION.getSymbol(), + () -> unencryptedClientFactory.createS3Client(getUri(), clientCreationParameters)); + } + /** * Create the function to create the Transfer Manager. * @return a callable which will create the component. @@ -182,6 +208,18 @@ public synchronized S3Client getOrCreateAsyncS3ClientUnchecked() throws Unchecke return s3Client.get(); } + /** + * Get or create an unencrypted S3 client. + * This is used for unencrypted operations when CSE is enabled with V1 compatibility. + * @return unencrypted S3 client + * @throws IOException on any failure + */ + @Override + public synchronized S3Client getOrCreateUnencryptedS3Client() throws IOException { + checkNotClosed(); + return unencryptedS3Client.eval(); + } + @Override public synchronized S3TransferManager getOrCreateTransferManager() throws IOException { checkNotClosed(); @@ -213,6 +251,7 @@ public synchronized void close() { l.add(closeAsync(transferManager)); l.add(closeAsync(s3AsyncClient)); l.add(closeAsync(s3Client)); + l.add(closeAsync(unencryptedS3Client)); // once all are queued, await their completion // and swallow any exception. @@ -261,6 +300,7 @@ public String toString() { "closed=" + closed.get() + ", s3Client=" + s3Client + ", s3AsyncClient=" + s3AsyncClient + + ", unencryptedS3Client=" + unencryptedS3Client + ", transferManager=" + transferManager + '}'; } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/EncryptionS3ClientFactory.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/EncryptionS3ClientFactory.java new file mode 100644 index 00000000000000..e89bb71445f58c --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/EncryptionS3ClientFactory.java @@ -0,0 +1,294 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.s3a.impl; + +import java.io.IOException; +import java.net.URI; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.s3a.DefaultS3ClientFactory; +import org.apache.hadoop.util.Preconditions; +import org.apache.hadoop.util.ReflectionUtils; +import org.apache.hadoop.util.functional.LazyAtomicReference; + +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.kms.KmsClient; +import software.amazon.awssdk.services.kms.KmsClientBuilder; +import software.amazon.awssdk.services.s3.S3AsyncClient; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.encryption.s3.S3AsyncEncryptionClient; +import software.amazon.encryption.s3.S3EncryptionClient; +import software.amazon.encryption.s3.materials.CryptographicMaterialsManager; +import software.amazon.encryption.s3.materials.DefaultCryptoMaterialsManager; +import software.amazon.encryption.s3.materials.Keyring; +import software.amazon.encryption.s3.materials.KmsKeyring; + +import static org.apache.hadoop.fs.s3a.impl.InstantiationIOException.unavailable; + +/** + * Factory class to create encrypted s3 client and encrypted async s3 client. + */ +public class EncryptionS3ClientFactory extends DefaultS3ClientFactory { + + /** + * Encryption client class name. + * value: {@value} + */ + private static final String ENCRYPTION_CLIENT_CLASSNAME = + "software.amazon.encryption.s3.S3EncryptionClient"; + + /** + * Encryption client availability. + */ + private static final LazyAtomicReference ENCRYPTION_CLIENT_AVAILABLE = + LazyAtomicReference.lazyAtomicReferenceFromSupplier( + EncryptionS3ClientFactory::checkForEncryptionClient + ); + + + /** + * S3Client to be wrapped by encryption client. + */ + private S3Client s3Client; + + /** + * S3AsyncClient to be wrapped by encryption client. + */ + private S3AsyncClient s3AsyncClient; + + /** + * Checks if {@link #ENCRYPTION_CLIENT_CLASSNAME} is available in the class path. + * @return true if available, false otherwise. + */ + private static boolean checkForEncryptionClient() { + try { + ClassLoader cl = EncryptionS3ClientFactory.class.getClassLoader(); + cl.loadClass(ENCRYPTION_CLIENT_CLASSNAME); + LOG.debug("encryption client class {} found", ENCRYPTION_CLIENT_CLASSNAME); + return true; + } catch (Exception e) { + LOG.debug("encryption client class {} not found", ENCRYPTION_CLIENT_CLASSNAME, e); + return false; + } + } + + /** + * Is the Encryption client available? + * @return true if it was found in the classloader + */ + private static synchronized boolean isEncryptionClientAvailable() { + return ENCRYPTION_CLIENT_AVAILABLE.get(); + } + + /** + * Creates both synchronous and asynchronous encrypted s3 clients. + * Synchronous client is wrapped by encryption client first and then + * Asynchronous client is wrapped by encryption client. + * @param uri S3A file system URI + * @param parameters parameter object + * @return encrypted s3 client + * @throws IOException IO failures + */ + @Override + public S3Client createS3Client(URI uri, S3ClientCreationParameters parameters) + throws IOException { + if (!isEncryptionClientAvailable()) { + throw unavailable(uri, ENCRYPTION_CLIENT_CLASSNAME, null, + "No encryption client available"); + } + + s3Client = super.createS3Client(uri, parameters); + s3AsyncClient = super.createS3AsyncClient(uri, parameters); + + return createS3EncryptionClient(parameters); + } + + /** + * Create async encrypted s3 client. + * @param uri S3A file system URI + * @param parameters parameter object + * @return async encrypted s3 client + * @throws IOException IO failures + */ + @Override + public S3AsyncClient createS3AsyncClient(URI uri, S3ClientCreationParameters parameters) + throws IOException { + if (!isEncryptionClientAvailable()) { + throw unavailable(uri, ENCRYPTION_CLIENT_CLASSNAME, null, + "No encryption client available"); + } + return createS3AsyncEncryptionClient(parameters); + } + + /** + * Creates an S3EncryptionClient instance based on the provided parameters. + * + * @param parameters The S3ClientCreationParameters containing the necessary configuration. + * @return An instance of S3EncryptionClient. + */ + private S3Client createS3EncryptionClient(S3ClientCreationParameters parameters) { + CSEMaterials cseMaterials = parameters.getClientSideEncryptionMaterials(); + Preconditions.checkArgument(s3AsyncClient != null, + "S3 async client not initialized"); + Preconditions.checkArgument(s3Client != null, + "S3 client not initialized"); + Preconditions.checkArgument(parameters != null, + "S3ClientCreationParameters is not initialized"); + + S3EncryptionClient.Builder s3EncryptionClientBuilder = + S3EncryptionClient.builder() + .wrappedAsyncClient(s3AsyncClient) + .wrappedClient(s3Client) + // this is required for doing S3 ranged GET calls + .enableLegacyUnauthenticatedModes(true) + // this is required for backward compatibility with older encryption clients + .enableLegacyWrappingAlgorithms(true); + + switch (cseMaterials.getCseKeyType()) { + case KMS: + Keyring kmsKeyring = createKmsKeyring(parameters, cseMaterials); + CryptographicMaterialsManager kmsCryptoMaterialsManager = + DefaultCryptoMaterialsManager.builder() + .keyring(kmsKeyring) + .build(); + s3EncryptionClientBuilder.cryptoMaterialsManager(kmsCryptoMaterialsManager); + break; + case CUSTOM: + Keyring keyring = getKeyringProvider(cseMaterials.getCustomKeyringClassName(), + cseMaterials.getConf()); + CryptographicMaterialsManager customCryptoMaterialsManager = + DefaultCryptoMaterialsManager.builder() + .keyring(keyring) + .build(); + s3EncryptionClientBuilder.cryptoMaterialsManager(customCryptoMaterialsManager); + break; + default: + break; + } + return s3EncryptionClientBuilder.build(); + } + + /** + * Creates KmsKeyring instance based on the provided S3ClientCreationParameters and CSEMaterials. + * + * @param parameters The S3ClientCreationParameters containing the necessary configuration. + * @param cseMaterials The CSEMaterials containing the KMS key ID and other encryption materials. + * @return A KmsKeyring instance configured with the appropriate KMS client and wrapping key ID. + */ + private Keyring createKmsKeyring(S3ClientCreationParameters parameters, + CSEMaterials cseMaterials) { + KmsClientBuilder kmsClientBuilder = KmsClient.builder(); + if (parameters.getCredentialSet() != null) { + kmsClientBuilder.credentialsProvider(parameters.getCredentialSet()); + } + if (parameters.getRegion() != null) { + kmsClientBuilder.region(Region.of(parameters.getRegion())); + } else if (parameters.getEndpoint() != null) { + String endpointStr = parameters.getEndpoint(); + URI endpoint = getS3Endpoint(endpointStr, cseMaterials.getConf()); + kmsClientBuilder.endpointOverride(endpoint); + } + return KmsKeyring.builder() + .kmsClient(kmsClientBuilder.build()) + .wrappingKeyId(cseMaterials.getKmsKeyId()) + .build(); + } + + /** + * Creates an S3AsyncEncryptionClient instance based on the provided parameters. + * + * @param parameters The S3ClientCreationParameters containing the necessary configuration. + * @return An instance of S3AsyncEncryptionClient. + */ + private S3AsyncClient createS3AsyncEncryptionClient(S3ClientCreationParameters parameters) { + Preconditions.checkArgument(s3AsyncClient != null, + "S3 async client not initialized"); + Preconditions.checkArgument(parameters != null, + "S3ClientCreationParameters is not initialized"); + + S3AsyncEncryptionClient.Builder s3EncryptionAsyncClientBuilder = + S3AsyncEncryptionClient.builder() + .wrappedClient(s3AsyncClient) + // this is required for doing S3 ranged GET calls + .enableLegacyUnauthenticatedModes(true) + // this is required for backward compatibility with older encryption clients + .enableLegacyWrappingAlgorithms(true); + + CSEMaterials cseMaterials = parameters.getClientSideEncryptionMaterials(); + switch (cseMaterials.getCseKeyType()) { + case KMS: + Keyring kmsKeyring = createKmsKeyring(parameters, cseMaterials); + CryptographicMaterialsManager kmsCryptoMaterialsManager = + DefaultCryptoMaterialsManager.builder() + .keyring(kmsKeyring) + .build(); + s3EncryptionAsyncClientBuilder.cryptoMaterialsManager(kmsCryptoMaterialsManager); + break; + case CUSTOM: + Keyring keyring = getKeyringProvider(cseMaterials.getCustomKeyringClassName(), + cseMaterials.getConf()); + CryptographicMaterialsManager customCryptoMaterialsManager = + DefaultCryptoMaterialsManager.builder() + .keyring(keyring) + .build(); + s3EncryptionAsyncClientBuilder.cryptoMaterialsManager(customCryptoMaterialsManager); + break; + default: + break; + } + return s3EncryptionAsyncClientBuilder.build(); + } + + + /** + * Retrieves an instance of the Keyring provider based on the provided class name. + * + * @param className The fully qualified class name of the Keyring provider implementation. + * @param conf The Configuration object containing the necessary configuration properties. + * @return An instance of the Keyring provider. + */ + private Keyring getKeyringProvider(String className, Configuration conf) { + try { + return ReflectionUtils.newInstance(getCustomKeyringProviderClass(className), conf); + } catch (Exception e) { + // this is for testing purpose to support CustomKeyring.java + return ReflectionUtils.newInstance(getCustomKeyringProviderClass(className), conf, + new Class[] {Configuration.class}, conf); + } + } + + /** + * Retrieves the Class object for the custom Keyring provider based on the provided class name. + * + * @param className The fully qualified class name of the custom Keyring provider implementation. + * @return The Class object representing the custom Keyring provider implementation. + * @throws IllegalArgumentException If the provided class name is null or empty, + * or if the specified class is not found. + */ + private Class getCustomKeyringProviderClass(String className) { + Preconditions.checkArgument(className !=null && !className.isEmpty(), + "Custom Keyring class name is null or empty"); + try { + return Class.forName(className).asSubclass(Keyring.class); + } catch (ClassNotFoundException e) { + throw new IllegalArgumentException( + "Custom CryptographicMaterialsManager class " + className + "not found", e); + } + } +} diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/HeaderProcessing.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/HeaderProcessing.java index 3865c391d6ddbb..2771e02bd6eb5d 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/HeaderProcessing.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/HeaderProcessing.java @@ -325,19 +325,6 @@ private Map retrieveHeaders( md.contentEncoding()); maybeSetHeader(headers, XA_CONTENT_LANGUAGE, md.contentLanguage()); - // If CSE is enabled, use the unencrypted content length. - // TODO: CSE is not supported yet, add these headers in during CSE work. -// if (md.getUserMetaDataOf(Headers.CRYPTO_CEK_ALGORITHM) != null -// && md.getUserMetaDataOf(Headers.UNENCRYPTED_CONTENT_LENGTH) != null) { -// maybeSetHeader(headers, XA_CONTENT_LENGTH, -// md.getUserMetaDataOf(Headers.UNENCRYPTED_CONTENT_LENGTH)); -// } else { -// maybeSetHeader(headers, XA_CONTENT_LENGTH, -// md.contentLength()); -// } -// maybeSetHeader(headers, XA_CONTENT_MD5, -// md.getContentMD5()); - // TODO: Add back in else block during CSE work. maybeSetHeader(headers, XA_CONTENT_LENGTH, md.contentLength()); if (md.sdkHttpResponse() != null && md.sdkHttpResponse().headers() != null diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/ListingOperationCallbacks.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/ListingOperationCallbacks.java index f4d7a4349e4846..200e9a386b3592 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/ListingOperationCallbacks.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/ListingOperationCallbacks.java @@ -21,6 +21,8 @@ import java.io.IOException; import java.util.concurrent.CompletableFuture; +import software.amazon.awssdk.services.s3.model.S3Object; + import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.s3a.Retries; @@ -105,6 +107,15 @@ S3ListRequest createListObjectsRequest( */ long getDefaultBlockSize(Path path); + /** + * Get the S3 object size. + * If the object is encrypted, the unpadded size will be returned. + * @param s3Object S3object + * @return plaintext S3 object size + * @throws IOException IO problems + */ + long getObjectSize(S3Object s3Object) throws IOException; + /** * Get the maximum key count. * @return a value, valid after initialization diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/RequestFactoryImpl.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/RequestFactoryImpl.java index e9f7d707286a81..00d9368aa58f11 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/RequestFactoryImpl.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/RequestFactoryImpl.java @@ -44,6 +44,7 @@ import software.amazon.awssdk.services.s3.model.MetadataDirective; import software.amazon.awssdk.services.s3.model.ObjectIdentifier; import software.amazon.awssdk.services.s3.model.PutObjectRequest; +import software.amazon.awssdk.services.s3.model.SdkPartType; import software.amazon.awssdk.services.s3.model.ServerSideEncryption; import software.amazon.awssdk.services.s3.model.StorageClass; import software.amazon.awssdk.services.s3.model.UploadPartRequest; @@ -588,6 +589,7 @@ public UploadPartRequest.Builder newUploadPartRequestBuilder( String destKey, String uploadId, int partNumber, + boolean isLastPart, long size) throws PathIOException { checkNotNull(uploadId); checkArgument(size >= 0, "Invalid partition size %s", size); @@ -609,6 +611,9 @@ public UploadPartRequest.Builder newUploadPartRequestBuilder( .uploadId(uploadId) .partNumber(partNumber) .contentLength(size); + if (isLastPart) { + builder.sdkPartType(SdkPartType.LAST); + } uploadPartEncryptionParameters(builder); // Set the request timeout for the part upload diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/S3AFileSystemHandler.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/S3AFileSystemHandler.java new file mode 100644 index 00000000000000..734bbcca07933c --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/S3AFileSystemHandler.java @@ -0,0 +1,125 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.s3a.impl; + +import java.io.IOException; + +import software.amazon.awssdk.core.ResponseInputStream; +import software.amazon.awssdk.services.s3.model.GetObjectRequest; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; +import software.amazon.awssdk.services.s3.model.HeadObjectResponse; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.s3a.Listing; +import org.apache.hadoop.fs.s3a.S3AEncryptionMethods; +import org.apache.hadoop.fs.s3a.S3AStore; +import org.apache.hadoop.fs.s3a.S3ClientFactory; +import org.apache.hadoop.fs.s3a.api.RequestFactory; +import org.apache.hadoop.fs.statistics.impl.IOStatisticsStore; + +/** + * An interface that defines the contract for handling certain filesystem operations. + */ +public interface S3AFileSystemHandler { + + /** + * Returns a {@link Listing.FileStatusAcceptor} object. + * That determines which files and directories should be included in a listing operation. + * + * @param path the path for which the listing is being performed + * @param includeSelf a boolean indicating whether the path itself should + * be included in the listing + * @return a {@link Listing.FileStatusAcceptor} object + */ + Listing.FileStatusAcceptor getFileStatusAcceptor(Path path, boolean includeSelf); + + /** + * Returns a {@link Listing.FileStatusAcceptor} object. + * That determines which files and directories should be included in a listing operation. + * + * @param path the path for which the listing is being performed + * @return a {@link Listing.FileStatusAcceptor} object + */ + Listing.FileStatusAcceptor getFileStatusAcceptor(Path path); + + /** + * Retrieves an object from the S3. + * + * @param store The S3AStore object representing the S3 bucket. + * @param request The GetObjectRequest containing the details of the object to retrieve. + * @param factory The RequestFactory used to create the GetObjectRequest. + * @return A ResponseInputStream containing the GetObjectResponse. + * @throws IOException If an error occurs while retrieving the object. + */ + ResponseInputStream getObject(S3AStore store, GetObjectRequest request, + RequestFactory factory) throws IOException; + + /** + * Set the client side encryption gauge to 0 or 1, indicating if CSE is enabled. + * @param ioStatisticsStore The IOStatisticsStore of the filesystem. + */ + void setCSEGauge(IOStatisticsStore ioStatisticsStore); + + /** + * Retrieves the client-side encryption materials for the given bucket and encryption algorithm. + * + * @param conf The Hadoop configuration object. + * @param bucket The name of the S3 bucket. + * @param algorithm The client-side encryption algorithm to use. + * @return The client-side encryption materials. + * @throws IOException If an error occurs while retrieving the encryption materials. + */ + CSEMaterials getClientSideEncryptionMaterials(Configuration conf, String bucket, + S3AEncryptionMethods algorithm) throws IOException; + + /** + * Retrieves the S3 client factory for the specified class and configuration. + * + * @param conf The Hadoop configuration object. + * @return The S3 client factory instance. + */ + S3ClientFactory getS3ClientFactory(Configuration conf); + + /** + * Retrieves the S3 client factory for the specified class and configuration. + * + * @param conf The Hadoop configuration object. + * @return The S3 client factory instance. + */ + S3ClientFactory getUnencryptedS3ClientFactory(Configuration conf); + + + /** + * Retrieves the size of a S3 object. + * + * @param key The key (path) of the object in the S3 bucket. + * @param length The expected length of the object, if known. If not known, pass -1. + * @param store The S3AStore object representing the S3 bucket. + * @param bucket The name of the S3 bucket. + * @param factory The RequestFactory used to create the HeadObjectRequest. + * @param response The HeadObjectResponse containing the metadata of the object. + * @return The size of the object in bytes. + * @throws IOException If an error occurs while retrieving the object size. + */ + long getS3ObjectSize(String key, long length, S3AStore store, String bucket, + RequestFactory factory, HeadObjectResponse response) throws IOException; + +} + diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/S3AMultipartUploader.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/S3AMultipartUploader.java index 58e38c2873bd0c..e00319c3dc581a 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/S3AMultipartUploader.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/S3AMultipartUploader.java @@ -139,6 +139,7 @@ public CompletableFuture startUpload( public CompletableFuture putPart( final UploadHandle uploadId, final int partNumber, + final boolean isLastPart, final Path filePath, final InputStream inputStream, final long lengthInBytes) @@ -154,7 +155,7 @@ public CompletableFuture putPart( return context.submit(new CompletableFuture<>(), () -> { UploadPartRequest request = writeOperations.newUploadPartRequestBuilder(key, - uploadIdString, partNumber, lengthInBytes).build(); + uploadIdString, partNumber, isLastPart, lengthInBytes).build(); RequestBody body = RequestBody.fromInputStream(inputStream, lengthInBytes); UploadPartResponse response = writeOperations.uploadPart(request, body, statistics); statistics.partPut(lengthInBytes); diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/S3AStoreImpl.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/S3AStoreImpl.java index 385023598c5596..38fb5a1e7f6424 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/S3AStoreImpl.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/S3AStoreImpl.java @@ -247,6 +247,11 @@ public S3Client getOrCreateAsyncS3ClientUnchecked() throws UncheckedIOException return clientManager.getOrCreateAsyncS3ClientUnchecked(); } + @Override + public S3Client getOrCreateUnencryptedS3Client() throws IOException { + return clientManager.getOrCreateUnencryptedS3Client(); + } + @Override public DurationTrackerFactory getDurationTrackerFactory() { return durationTrackerFactory; diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/encryption.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/encryption.md index 7b9e8d0412efd7..674e987e278ede 100644 --- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/encryption.md +++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/encryption.md @@ -680,9 +680,9 @@ client-side and then transmit it over to S3 storage. The same encrypted data is then transmitted over to client while reading and then decrypted on the client-side. -S3-CSE, uses `AmazonS3EncryptionClientV2.java` as the AmazonS3 client. The -encryption and decryption is done by AWS SDK. As of July 2021, Only CSE-KMS -method is supported. +S3-CSE, uses `S3EncryptionClient.java` (V3) as the AmazonS3 client. The +encryption and decryption is done by AWS SDK. Both CSE-KMS and CSE-CUSTOM +methods are supported. A key reason this feature (HADOOP-13887) has been unavailable for a long time is that the AWS S3 client pads uploaded objects with a 16 byte footer. This @@ -704,10 +704,20 @@ clients where S3-CSE has not been enabled. ### Features -- Supports client side encryption with keys managed in AWS KMS. +- Supports client side encryption with keys managed in AWS KMS (CSE-KMS) +- Supports client side encryption with custom keys by implementing custom [Keyring](https://docs.aws.amazon.com/encryption-sdk/latest/developer-guide/choose-keyring.html) (CSE-CUSTOM) +- Backward compatible with older encryption clients like `AmazonS3EncryptionClient.java`(V1) and `AmazonS3EncryptionClientV2.java`(V2) - encryption settings propagated into jobs through any issued delegation tokens. - encryption information stored as headers in the uploaded object. +### Compatibility Issues + +- The V1 and V2 clients support reading unencrypted S3 objects, whereas the V3 client does not. In order to read S3 objects in a directory with a mix of encrypted and unencrypted objects. +- Unlike the V2 and V3 clients which always pads 16 bytes, V1 client pads extra bytes to the next multiple of 16. For example if unencrypted object size is 12bytes, V1 client pads extra 4bytes to make it multiple of 16. +- The V1 client supports storing encryption metadata in instruction file. + +Inorder to workaround the above compatibility issues set `fs.s3a.encryption.cse.v1.compatibility.enabled=true` + ### Limitations - Performance will be reduced. All encrypt/decrypt is now being done on the @@ -722,6 +732,7 @@ clients where S3-CSE has not been enabled. NIST. ### Setup +#### 1. CSE-KMS - Generate an AWS KMS Key ID from AWS console for your bucket, with same region as the storage bucket. - If already created, [view the kms key ID by these steps.](https://docs.aws.amazon.com/kms/latest/developerguide/find-cmk-id-arn.html) @@ -757,6 +768,28 @@ S3-CSE to work. ``` +#### 2. CSE-CUSTOM +- Set `fs.s3a.encryption.algorithm=CSE-CUSTOM`. +- Set `fs.s3a.encryption.cse.custom.cryptographic.material.manager.class.name=`. + +Example for custom keyring implementation +``` +public class CustomKeyring implements Keyring { + public CustomKeyring() { + } + + @Override + public EncryptionMaterials onEncrypt(EncryptionMaterials encryptionMaterials) { + // custom code + } + + @Override + public DecryptionMaterials onDecrypt(DecryptionMaterials decryptionMaterials, + List list) { + // custom code + } +``` + ## Troubleshooting Encryption The [troubleshooting](./troubleshooting_s3a.html) document covers diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/CustomKeyring.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/CustomKeyring.java new file mode 100644 index 00000000000000..bbc0bd3d5d1eb0 --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/CustomKeyring.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.s3a; + +import java.io.IOException; +import java.util.List; + +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.kms.KmsClient; +import software.amazon.encryption.s3.materials.DecryptionMaterials; +import software.amazon.encryption.s3.materials.EncryptedDataKey; +import software.amazon.encryption.s3.materials.EncryptionMaterials; +import software.amazon.encryption.s3.materials.Keyring; +import software.amazon.encryption.s3.materials.KmsKeyring; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; + +import static org.apache.hadoop.fs.s3a.Constants.AWS_REGION; +import static org.apache.hadoop.fs.s3a.Constants.AWS_S3_DEFAULT_REGION; + +/** + * Custom Keyring implementation by warpping over KmsKeyring. + * This is used for testing {@link ITestS3AClientSideEncryptionCustom}. + */ +public class CustomKeyring implements Keyring { + private final KmsClient kmsClient; + private final Configuration conf; + private final KmsKeyring kmsKeyring; + + + public CustomKeyring(Configuration conf) throws IOException { + this.conf = conf; + String bucket = S3ATestUtils.getFsName(conf); + kmsClient = KmsClient.builder() + .region(Region.of(conf.get(AWS_REGION, AWS_S3_DEFAULT_REGION))) + .credentialsProvider(new TemporaryAWSCredentialsProvider( + new Path(bucket).toUri(), conf)) + .build(); + kmsKeyring = KmsKeyring.builder() + .kmsClient(kmsClient) + .wrappingKeyId(S3AUtils.getS3EncryptionKey(bucket, conf)) + .build(); + } + + @Override + public EncryptionMaterials onEncrypt(EncryptionMaterials encryptionMaterials) { + return kmsKeyring.onEncrypt(encryptionMaterials); + } + + @Override + public DecryptionMaterials onDecrypt(DecryptionMaterials decryptionMaterials, + List list) { + return kmsKeyring.onDecrypt(decryptionMaterials, list); + } +} diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AClientSideEncryption.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AClientSideEncryption.java index 4094b22eb19267..c8436502d8a902 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AClientSideEncryption.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AClientSideEncryption.java @@ -21,10 +21,13 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; +import java.util.HashMap; import java.util.List; +import java.util.Map; import org.assertj.core.api.Assertions; import org.junit.Test; +import software.amazon.awssdk.services.s3.model.PutObjectRequest; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataInputStream; @@ -34,8 +37,13 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.RemoteIterator; import org.apache.hadoop.fs.contract.ContractTestUtils; +import org.apache.hadoop.fs.s3a.api.RequestFactory; +import org.apache.hadoop.fs.s3a.impl.AWSHeaders; +import org.apache.hadoop.fs.s3a.impl.PutObjectOptions; +import org.apache.hadoop.fs.s3a.impl.RequestFactoryImpl; import org.apache.hadoop.fs.statistics.IOStatisticAssertions; import org.apache.hadoop.fs.statistics.IOStatistics; +import org.apache.hadoop.fs.store.audit.AuditSpan; import static org.apache.hadoop.fs.contract.ContractTestUtils.createFile; import static org.apache.hadoop.fs.contract.ContractTestUtils.dataset; @@ -44,10 +52,13 @@ import static org.apache.hadoop.fs.contract.ContractTestUtils.writeDataset; import static org.apache.hadoop.fs.s3a.Constants.MULTIPART_MIN_SIZE; import static org.apache.hadoop.fs.s3a.Constants.S3_ENCRYPTION_ALGORITHM; +import static org.apache.hadoop.fs.s3a.Constants.S3_ENCRYPTION_CSE_V1_COMPATIBILITY_ENABLED; +import static org.apache.hadoop.fs.s3a.Constants.S3_ENCRYPTION_CSE_INSTRUCTION_FILE_SUFFIX; import static org.apache.hadoop.fs.s3a.Constants.S3_ENCRYPTION_KEY; import static org.apache.hadoop.fs.s3a.Constants.SERVER_SIDE_ENCRYPTION_ALGORITHM; import static org.apache.hadoop.fs.s3a.Constants.SERVER_SIDE_ENCRYPTION_KEY; import static org.apache.hadoop.fs.s3a.S3ATestUtils.assume; +import static org.apache.hadoop.fs.s3a.S3ATestUtils.createTestFileSystem; import static org.apache.hadoop.fs.s3a.S3ATestUtils.getTestBucketName; import static org.apache.hadoop.fs.s3a.S3ATestUtils.getTestPropertyBool; import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides; @@ -232,13 +243,8 @@ public void testEncryptionEnabledAndDisabledFS() throws Exception { // CSE enabled FS trying to read unencrypted data would face an exception. try (FSDataInputStream in = cseEnabledFS.open(unEncryptedFilePath)) { - FileStatus encryptedFSFileStatus = - cseEnabledFS.getFileStatus(unEncryptedFilePath); - assertEquals("Mismatch in content length bytes", SMALL_FILE_SIZE, - encryptedFSFileStatus.getLen()); - - intercept(SecurityException.class, "", - "SecurityException should be thrown", + intercept(AWSClientIOException.class, "Instruction file not found!", + "AWSClientIOException should be thrown", () -> { in.read(new byte[SMALL_FILE_SIZE]); return "Exception should be raised if unencrypted data is read by " @@ -266,6 +272,211 @@ public void testEncryptionEnabledAndDisabledFS() throws Exception { } } + /** + * Test to check if unencrypted objects are read with V1 client compatibility. + * @throws IOException + * @throws Exception + */ + @Test + public void testUnencryptedObjectReadWithV1CompatibilityConfig() throws Exception { + maybeSkipTest(); + // initialize base s3 client. + Configuration conf = new Configuration(getConfiguration()); + S3AFileSystem nonCseFs = createTestFileSystem(conf); + removeBaseAndBucketOverrides(getTestBucketName(conf), + conf, + S3_ENCRYPTION_ALGORITHM, + S3_ENCRYPTION_KEY, + SERVER_SIDE_ENCRYPTION_ALGORITHM, + SERVER_SIDE_ENCRYPTION_KEY); + nonCseFs.initialize(getFileSystem().getUri(), conf); + + Path file = path(getMethodName()); + // write unencrypted file + ContractTestUtils.writeDataset(nonCseFs, file, new byte[SMALL_FILE_SIZE], + SMALL_FILE_SIZE, SMALL_FILE_SIZE, true); + + Configuration cseConf = new Configuration(getConfiguration()); + cseConf.setBoolean(S3_ENCRYPTION_CSE_V1_COMPATIBILITY_ENABLED, true); + // create filesystem with cse enabled and v1 compatibility. + S3AFileSystem cseFs = createTestFileSystem(cseConf); + cseFs.initialize(getFileSystem().getUri(), cseConf); + + // read unencrypted file. It should not throw any exception. + try (FSDataInputStream in = cseFs.open(file)) { + in.read(new byte[SMALL_FILE_SIZE]); + } finally { + // close the filesystem + nonCseFs.close(); + cseFs.close(); + } + } + + /** + * Test to check if file name with suffix ".instruction" is ignored with V1 compatibility. + * @throws IOException + */ + @Test + public void testSkippingCSEInstructionFileWithV1Compatibility() throws IOException { + maybeSkipTest(); + // initialize base s3 client. + Configuration conf = new Configuration(getConfiguration()); + S3AFileSystem fs = createTestFileSystem(conf); + removeBaseAndBucketOverrides(getTestBucketName(conf), + conf, + S3_ENCRYPTION_ALGORITHM, + S3_ENCRYPTION_KEY, + SERVER_SIDE_ENCRYPTION_ALGORITHM, + SERVER_SIDE_ENCRYPTION_KEY); + fs.initialize(getFileSystem().getUri(), conf); + + // write file with suffix ".instruction" + Path filePath = path(getMethodName()); + Path file = new Path(filePath, + "file" + S3_ENCRYPTION_CSE_INSTRUCTION_FILE_SUFFIX); + ContractTestUtils.writeDataset(fs, file, new byte[SMALL_FILE_SIZE], + SMALL_FILE_SIZE, SMALL_FILE_SIZE, true); + + // create filesystem with cse enabled and v1 compatibility. + Configuration cseConf = new Configuration(getConfiguration()); + cseConf.setBoolean(S3_ENCRYPTION_CSE_V1_COMPATIBILITY_ENABLED, true); + S3AFileSystem cseFs = createTestFileSystem(cseConf); + cseFs.initialize(getFileSystem().getUri(), cseConf); + try { + // listing from fs without cse + Assertions.assertThat(fs.listStatus(filePath)).describedAs( + "Number of files aren't the same " + "as expected from FileStatus dir") + .hasSize(1); + + // listing fs without cse with v1 compatibility + Assertions.assertThat(cseFs.listStatus(filePath)).describedAs( + "Number of files aren't the same " + "as expected from FileStatus dir") + .hasSize(0); + } finally { + // close the filesystem + fs.close(); + cseFs.close(); + } + } + + /** + * Tests the size of an encrypted object when with V1 compatibility and custom header length. + * + * @throws Exception If any error occurs during the test execution. + */ + @Test + public void testSizeOfEncryptedObjectFromHeaderWithV1Compatibility() throws Exception { + maybeSkipTest(); + Configuration cseConf = new Configuration(getConfiguration()); + cseConf.setBoolean(S3_ENCRYPTION_CSE_V1_COMPATIBILITY_ENABLED, true); + S3AFileSystem fs = createTestFileSystem(cseConf); + fs.initialize(getFileSystem().getUri(), cseConf); + + Path filePath = path(getMethodName()); + Path file = new Path(filePath, "file"); + String key = fs.pathToKey(file); + + + // write object with random content length header + Map metadata = new HashMap<>(); + metadata.put(AWSHeaders.UNENCRYPTED_CONTENT_LENGTH, "10"); + try (AuditSpan span = span()) { + RequestFactory factory = RequestFactoryImpl.builder() + .withBucket(fs.getBucket()) + .build(); + PutObjectRequest.Builder putObjectRequestBuilder = + factory.newPutObjectRequestBuilder(key, null, SMALL_FILE_SIZE, + false); + putObjectRequestBuilder.contentLength(Long.parseLong(String.valueOf(SMALL_FILE_SIZE))); + putObjectRequestBuilder.metadata(metadata); + fs.putObjectDirect(putObjectRequestBuilder.build(), + PutObjectOptions.deletingDirs(), + new S3ADataBlocks.BlockUploadData(new byte[SMALL_FILE_SIZE], null), + null); + + // fetch the random content length + long contentLength = fs.getFileStatus(file).getLen(); + assertEquals("content length does not match", 10, contentLength); + } finally { + fs.close(); + } + } + + /** + * Tests the size of an unencrypted object when using V1 compatibility mode. + * + * @throws Exception If any error occurs during the test execution. + */ + @Test + public void testSizeOfUnencryptedObjectWithV1Compatibility() throws Exception { + maybeSkipTest(); + // initialize base s3 client. + Configuration conf = new Configuration(getConfiguration()); + S3AFileSystem fs = createTestFileSystem(conf); + removeBaseAndBucketOverrides(getTestBucketName(conf), + conf, + S3_ENCRYPTION_ALGORITHM, + S3_ENCRYPTION_KEY, + SERVER_SIDE_ENCRYPTION_ALGORITHM, + SERVER_SIDE_ENCRYPTION_KEY); + fs.initialize(getFileSystem().getUri(), conf); + + Path file = path(getMethodName()); + // Unencrypted data written to a path. + ContractTestUtils.writeDataset(fs, file, new byte[SMALL_FILE_SIZE], + SMALL_FILE_SIZE, SMALL_FILE_SIZE, true); + + // initialize encrypted s3 client with support for reading unencrypted objects + Configuration cseConf = new Configuration(getConfiguration()); + cseConf.setBoolean(S3_ENCRYPTION_CSE_V1_COMPATIBILITY_ENABLED, true); + S3AFileSystem cseFs = createTestFileSystem(cseConf); + cseFs.initialize(getFileSystem().getUri(), cseConf); + + // check the file size + try { + FileStatus status1 = fs.getFileStatus(file); + assertEquals("Mismatch in content length bytes", SMALL_FILE_SIZE, + status1.getLen()); + FileStatus status2 = cseFs.getFileStatus(file); + assertEquals("Mismatch in content length bytes", SMALL_FILE_SIZE, + status2.getLen()); + } finally { + // close the filesystem + fs.close(); + cseFs.close(); + } + } + + /** + * Tests the size of an encrypted object when using V1 compatibility mode. + * + * @throws Exception If any error occurs during the test execution. + */ + @Test + public void testSizeOfEncryptedObjectWithV1Compatibility() throws Exception { + maybeSkipTest(); + Configuration cseConf = new Configuration(getConfiguration()); + cseConf.setBoolean(S3_ENCRYPTION_CSE_V1_COMPATIBILITY_ENABLED, true); + S3AFileSystem fs = createTestFileSystem(cseConf); + fs.initialize(getFileSystem().getUri(), cseConf); + + // write encrypted file + Path file = path(getMethodName()); + ContractTestUtils.writeDataset(fs, file, new byte[SMALL_FILE_SIZE], + SMALL_FILE_SIZE, SMALL_FILE_SIZE, true); + + // check the file size + try { + FileStatus status2 = fs.getFileStatus(file); + assertEquals("Mismatch in content length bytes", SMALL_FILE_SIZE, + status2.getLen()); + } finally { + // close the filesystem + fs.close(); + } + } + + @Override protected Configuration createConfiguration() { Configuration conf = super.createConfiguration(); diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AClientSideEncryptionCustom.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AClientSideEncryptionCustom.java new file mode 100644 index 00000000000000..c94b804a1ec566 --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AClientSideEncryptionCustom.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.s3a; + +import java.io.IOException; +import java.util.Map; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.s3a.impl.AWSHeaders; +import org.apache.hadoop.fs.s3a.impl.HeaderProcessing; + +import static org.apache.hadoop.fs.s3a.Constants.S3_ENCRYPTION_CSE_CUSTOM_KEYRING_CLASS_NAME; +import static org.apache.hadoop.fs.s3a.S3ATestUtils.skipIfEncryptionNotSet; +import static org.apache.hadoop.fs.s3a.S3ATestUtils.skipIfEncryptionTestsDisabled; + +/** + * Tests to verify S3 client side encryption (CSE-CUSTOM). + */ +public class ITestS3AClientSideEncryptionCustom extends ITestS3AClientSideEncryption { + + private static final String KMS_KEY_WRAP_ALGO = "kms+context"; + /** + * Creating custom configs for CSE-CUSTOM testing. + * + * @return Configuration. + */ + @Override + protected Configuration createConfiguration() { + Configuration conf = super.createConfiguration(); + S3ATestUtils.disableFilesystemCaching(conf); + conf.set(S3_ENCRYPTION_CSE_CUSTOM_KEYRING_CLASS_NAME, + CustomKeyring.class.getCanonicalName()); + return conf; + } + + @Override + protected void maybeSkipTest() throws IOException { + skipIfEncryptionTestsDisabled(getConfiguration()); + // skip the test if CSE-CUSTOM is not set. + skipIfEncryptionNotSet(getConfiguration(), S3AEncryptionMethods.CSE_CUSTOM); + } + + + @Override + protected void assertEncrypted(Path path) throws IOException { + Map fsXAttrs = getFileSystem().getXAttrs(path); + String xAttrPrefix = "header."; + + // Assert KeyWrap Algo + assertEquals("Key wrap algo isn't same as expected", KMS_KEY_WRAP_ALGO, + processHeader(fsXAttrs, + xAttrPrefix + AWSHeaders.CRYPTO_KEYWRAP_ALGORITHM)); + } + + /** + * A method to process a FS xAttribute Header key by decoding it. + * + * @param fsXAttrs Map of String(Key) and bytes[](Value) to represent fs + * xAttr. + * @param headerKey Key value of the header we are trying to process. + * @return String representing the value of key header provided. + */ + private String processHeader(Map fsXAttrs, + String headerKey) { + return HeaderProcessing.decodeBytes(fsXAttrs.get(headerKey)); + } +} diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java index 967ba885bc90fa..bfabcb2dff0120 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java @@ -32,6 +32,7 @@ import org.junit.rules.Timeout; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import software.amazon.awssdk.core.SdkClient; import software.amazon.awssdk.core.client.config.SdkClientConfiguration; import software.amazon.awssdk.core.client.config.SdkClientOption; import software.amazon.awssdk.core.interceptor.ExecutionAttributes; @@ -42,6 +43,7 @@ import software.amazon.awssdk.services.s3.model.HeadBucketRequest; import software.amazon.awssdk.services.sts.StsClient; import software.amazon.awssdk.services.sts.model.StsException; +import software.amazon.encryption.s3.S3EncryptionClient; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.reflect.FieldUtils; @@ -370,6 +372,7 @@ public void shouldBeAbleToSwitchOnS3PathStyleAccessViaConfigProperty() conf = new Configuration(); skipIfCrossRegionClient(conf); + unsetEncryption(conf); conf.set(Constants.PATH_STYLE_ACCESS, Boolean.toString(true)); assertTrue(conf.getBoolean(Constants.PATH_STYLE_ACCESS, false)); @@ -411,6 +414,7 @@ public void shouldBeAbleToSwitchOnS3PathStyleAccessViaConfigProperty() public void testDefaultUserAgent() throws Exception { conf = new Configuration(); skipIfCrossRegionClient(conf); + unsetEncryption(conf); fs = S3ATestUtils.createTestFileSystem(conf); assertNotNull(fs); S3Client s3 = getS3Client("User Agent"); @@ -425,6 +429,7 @@ public void testDefaultUserAgent() throws Exception { public void testCustomUserAgent() throws Exception { conf = new Configuration(); skipIfCrossRegionClient(conf); + unsetEncryption(conf); conf.set(Constants.USER_AGENT_PREFIX, "MyApp"); fs = S3ATestUtils.createTestFileSystem(conf); assertNotNull(fs); @@ -446,7 +451,10 @@ public void testRequestTimeout() throws Exception { Duration timeout = Duration.ofSeconds(120); conf.set(REQUEST_TIMEOUT, timeout.getSeconds() + "s"); fs = S3ATestUtils.createTestFileSystem(conf); - S3Client s3 = getS3Client("Request timeout (ms)"); + SdkClient s3 = getS3Client("Request timeout (ms)"); + if (s3 instanceof S3EncryptionClient) { + s3 = ((S3EncryptionClient) s3).delegate(); + } SdkClientConfiguration clientConfiguration = getField(s3, SdkClientConfiguration.class, "clientConfiguration"); Assertions.assertThat(clientConfiguration.option(SdkClientOption.API_CALL_ATTEMPT_TIMEOUT)) @@ -638,4 +646,15 @@ private static void skipIfCrossRegionClient( skip("Skipping test as cross region client is in use "); } } + + /** + * Unset encryption options. + * This is needed to avoid encryption tests interfering with non-encryption + * tests. + * @param conf Configuations + */ + private static void unsetEncryption(Configuration conf) { + removeBaseAndBucketOverrides(conf, S3_ENCRYPTION_ALGORITHM); + conf.set(Constants.S3_ENCRYPTION_ALGORITHM, S3AEncryptionMethods.NONE.getMethod()); + } } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ADelayedFNF.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ADelayedFNF.java index ca9d185c3e9e11..bad4d24e4d8a54 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ADelayedFNF.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ADelayedFNF.java @@ -24,6 +24,7 @@ import org.apache.hadoop.fs.contract.ContractTestUtils; import org.apache.hadoop.fs.s3a.impl.ChangeDetectionPolicy; import org.apache.hadoop.fs.s3a.impl.ChangeDetectionPolicy.Source; +import org.apache.hadoop.fs.s3a.impl.CSEUtils; import org.apache.hadoop.test.LambdaTestUtils; import org.junit.Assume; @@ -35,7 +36,9 @@ import static org.apache.hadoop.fs.s3a.Constants.CHANGE_DETECT_SOURCE; import static org.apache.hadoop.fs.s3a.Constants.RETRY_INTERVAL; import static org.apache.hadoop.fs.s3a.Constants.RETRY_LIMIT; +import static org.apache.hadoop.fs.s3a.S3ATestUtils.getTestBucketName; import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides; +import static org.apache.hadoop.fs.s3a.S3AUtils.getEncryptionAlgorithm; /** * Tests behavior of a FileNotFound error that happens after open(), i.e. on @@ -78,8 +81,14 @@ public void testNotFoundFirstRead() throws Exception { final FSDataInputStream in = fs.open(p); assertDeleted(p, false); + Class exceptionClass = FileNotFoundException.class; + if (CSEUtils.isCSEEnabled(getEncryptionAlgorithm( + getTestBucketName(getConfiguration()), getConfiguration()).getMethod())) { + exceptionClass = AWSClientIOException.class; + } + // This should fail since we deleted after the open. - LambdaTestUtils.intercept(FileNotFoundException.class, + LambdaTestUtils.intercept(exceptionClass, () -> in.read()); } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionSSEKMSUserDefinedKey.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionSSEKMSUserDefinedKey.java index 2be3fe88896246..2c191a2d1865f0 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionSSEKMSUserDefinedKey.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionSSEKMSUserDefinedKey.java @@ -22,6 +22,7 @@ import org.apache.hadoop.conf.Configuration; import static org.apache.hadoop.fs.contract.ContractTestUtils.skip; +import static org.apache.hadoop.fs.s3a.Constants.S3_ENCRYPTION_ALGORITHM; import static org.apache.hadoop.fs.s3a.Constants.S3_ENCRYPTION_KEY; import static org.apache.hadoop.fs.s3a.S3AEncryptionMethods.SSE_KMS; import static org.apache.hadoop.fs.s3a.S3ATestUtils.getTestBucketName; @@ -40,7 +41,8 @@ protected Configuration createConfiguration() { Configuration c = new Configuration(); String kmsKey = S3AUtils.getS3EncryptionKey(getTestBucketName(c), c); // skip the test if SSE-KMS or KMS key not set. - if (StringUtils.isBlank(kmsKey)) { + if (StringUtils.isBlank(kmsKey) || !SSE_KMS.getMethod() + .equals(c.get(S3_ENCRYPTION_ALGORITHM))) { skip(S3_ENCRYPTION_KEY + " is not set for " + SSE_KMS.getMethod()); } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEndpointRegion.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEndpointRegion.java index 80b061de03183d..07caeb02f416ae 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEndpointRegion.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEndpointRegion.java @@ -52,6 +52,7 @@ import static org.apache.hadoop.fs.s3a.Constants.ENDPOINT; import static org.apache.hadoop.fs.s3a.Constants.FIPS_ENDPOINT; import static org.apache.hadoop.fs.s3a.Constants.PATH_STYLE_ACCESS; +import static org.apache.hadoop.fs.s3a.Constants.S3_ENCRYPTION_ALGORITHM; import static org.apache.hadoop.fs.s3a.DefaultS3ClientFactory.ERROR_ENDPOINT_WITH_FIPS; import static org.apache.hadoop.fs.s3a.S3ATestUtils.assume; import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides; @@ -483,7 +484,8 @@ public void testCentralEndpointAndNullRegionWithCRUD() throws Throwable { newConf, ENDPOINT, AWS_REGION, - FIPS_ENDPOINT); + FIPS_ENDPOINT, + S3_ENCRYPTION_ALGORITHM); newConf.set(ENDPOINT, CENTRAL_ENDPOINT); diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFailureHandling.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFailureHandling.java index b550fc5864b737..be7c25644f9dff 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFailureHandling.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFailureHandling.java @@ -28,6 +28,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.RemoteIterator; import org.apache.hadoop.fs.s3a.impl.MultiObjectDeleteException; +import org.apache.hadoop.fs.s3a.impl.CSEUtils; import org.apache.hadoop.fs.statistics.StoreStatisticNames; import org.apache.hadoop.fs.store.audit.AuditSpan; @@ -41,8 +42,10 @@ import static org.apache.hadoop.fs.contract.ContractTestUtils.*; import static org.apache.hadoop.fs.s3a.S3ATestUtils.createFiles; +import static org.apache.hadoop.fs.s3a.S3ATestUtils.getTestBucketName; import static org.apache.hadoop.fs.s3a.S3ATestUtils.isBulkDeleteEnabled; import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides; +import static org.apache.hadoop.fs.s3a.S3AUtils.getEncryptionAlgorithm; import static org.apache.hadoop.fs.s3a.test.ExtraAssertions.failIf; import static org.apache.hadoop.fs.s3a.test.PublicDatasetTestUtils.isUsingDefaultExternalDataFile; import static org.apache.hadoop.fs.s3a.test.PublicDatasetTestUtils.requireDefaultExternalData; @@ -200,9 +203,14 @@ public void testSingleObjectDeleteNoPermissionsTranslated() throws Throwable { Path path = requireDefaultExternalData(getConfiguration()); S3AFileSystem fs = (S3AFileSystem) path.getFileSystem( getConfiguration()); - AccessDeniedException aex = intercept(AccessDeniedException.class, + Class exceptionClass = AccessDeniedException.class; + if (CSEUtils.isCSEEnabled(getEncryptionAlgorithm( + getTestBucketName(getConfiguration()), getConfiguration()).getMethod())) { + exceptionClass = AWSClientIOException.class; + } + Exception ex = (Exception) intercept(exceptionClass, () -> fs.delete(path, false)); - Throwable cause = aex.getCause(); - failIf(cause == null, "no nested exception", aex); + Throwable cause = ex.getCause(); + failIf(cause == null, "no nested exception", ex); } } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/MultipartTestUtils.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/MultipartTestUtils.java index 20e5c12ec79cf8..0c6acb4bb08f84 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/MultipartTestUtils.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/MultipartTestUtils.java @@ -92,7 +92,7 @@ public static IdKey createPartUpload(S3AFileSystem fs, String key, int len, InputStream in = new ByteArrayInputStream(data); String uploadId = writeHelper.initiateMultiPartUpload(key, PutObjectOptions.keepingDirs()); UploadPartRequest req = writeHelper.newUploadPartRequestBuilder(key, uploadId, - partNo, len).build(); + partNo, true, len).build(); RequestBody body = RequestBody.fromInputStream(in, len); UploadPartResponse response = writeHelper.uploadPart(req, body, null); LOG.debug("uploaded part etag {}, upid {}", response.eTag(), uploadId); diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3ABlockOutputStream.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3ABlockOutputStream.java index 5caa7c87855348..dc1cb2f54bfa65 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3ABlockOutputStream.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3ABlockOutputStream.java @@ -107,11 +107,11 @@ public void testWriteOperationHelperPartLimits() throws Throwable { // first one works String key = "destKey"; woh.newUploadPartRequestBuilder(key, - "uploadId", 1, 1024); + "uploadId", 1, false, 1024); // but ask past the limit and a PathIOE is raised intercept(PathIOException.class, key, () -> woh.newUploadPartRequestBuilder(key, - "uploadId", 50000, 1024)); + "uploadId", 50000, true, 1024)); } static class StreamClosedException extends ClosedIOException { diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/integration/ITestS3ACommitterMRJob.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/integration/ITestS3ACommitterMRJob.java index 71be373b407ed3..7488de41ce6386 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/integration/ITestS3ACommitterMRJob.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/integration/ITestS3ACommitterMRJob.java @@ -34,6 +34,7 @@ import java.util.UUID; import java.util.stream.Collectors; +import org.apache.hadoop.fs.s3a.Constants; import org.apache.hadoop.util.Sets; import org.assertj.core.api.Assertions; import org.junit.FixMethodOrder; @@ -80,6 +81,7 @@ import static org.apache.hadoop.fs.s3a.commit.InternalCommitterConstants.FS_S3A_COMMITTER_UUID; import static org.apache.hadoop.fs.s3a.commit.staging.Paths.getMultipartUploadCommitsDirectory; import static org.apache.hadoop.fs.s3a.commit.staging.StagingCommitterConstants.STAGING_UPLOADS; +import static org.apache.hadoop.mapred.JobConf.MAPRED_TASK_ENV; /** * Test an MR Job with all the different committers. @@ -338,6 +340,8 @@ public void test_200_execute() throws Exception { @Override protected void applyCustomConfigOptions(final JobConf jobConf) throws IOException { + jobConf.set(MAPRED_TASK_ENV, "AWS_REGION=" + jobConf.get(Constants.AWS_REGION)); + jobConf.set("yarn.app.mapreduce.am.env", "AWS_REGION=" + jobConf.get(Constants.AWS_REGION)); committerTestBinding.applyCustomConfigOptions(jobConf); } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContextStatistics.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContextStatistics.java index fbad671e1fa66f..1724006a831985 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContextStatistics.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/ITestS3AFileContextStatistics.java @@ -13,7 +13,6 @@ */ package org.apache.hadoop.fs.s3a.fileContext; -import java.io.IOException; import java.net.URI; import org.slf4j.Logger; @@ -24,7 +23,6 @@ import org.apache.hadoop.fs.FileContext; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.s3a.S3AEncryptionMethods; import org.apache.hadoop.fs.s3a.S3ATestUtils; import org.apache.hadoop.fs.s3a.auth.STSClientFactory; @@ -32,12 +30,6 @@ import org.junit.Assert; import org.junit.Before; -import static org.apache.hadoop.fs.s3a.S3ATestConstants.KMS_KEY_GENERATION_REQUEST_PARAMS_BYTES_WRITTEN; -import static org.apache.hadoop.fs.s3a.S3ATestUtils.getTestBucketName; -import static org.apache.hadoop.fs.s3a.S3AUtils.getEncryptionAlgorithm; -import static org.apache.hadoop.fs.s3a.S3AUtils.getS3EncryptionKey; -import static org.apache.hadoop.fs.s3a.impl.InternalConstants.CSE_PADDING_LENGTH; - /** * S3a implementation of FCStatisticsBaseTest. */ @@ -73,30 +65,12 @@ protected void verifyReadBytes(FileSystem.Statistics stats) { /** * A method to verify the bytes written. - *
- * NOTE: if Client side encryption is enabled, expected bytes written - * should increase by 16(padding of data) + bytes for the key ID set + 94(KMS - * key generation) in case of storage type CryptoStorageMode as - * ObjectMetadata(Default). If Crypto Storage mode is instruction file then - * add additional bytes as that file is stored separately and would account - * for bytes written. - * * @param stats Filesystem statistics. */ @Override - protected void verifyWrittenBytes(FileSystem.Statistics stats) - throws IOException { + protected void verifyWrittenBytes(FileSystem.Statistics stats) { //No extra bytes are written - long expectedBlockSize = blockSize; - if (S3AEncryptionMethods.CSE_KMS.getMethod() - .equals(getEncryptionAlgorithm(getTestBucketName(conf), conf) - .getMethod())) { - String keyId = getS3EncryptionKey(getTestBucketName(conf), conf); - // Adding padding length and KMS key generation bytes written. - expectedBlockSize += CSE_PADDING_LENGTH + keyId.getBytes().length + - KMS_KEY_GENERATION_REQUEST_PARAMS_BYTES_WRITTEN; - } - Assert.assertEquals("Mismatch in bytes written", expectedBlockSize, + Assert.assertEquals("Mismatch in bytes written", blockSize, stats.getBytesWritten()); } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/ITestConnectionTimeouts.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/ITestConnectionTimeouts.java index 9ca12e4f31a602..362c8a8ac1735b 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/ITestConnectionTimeouts.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/ITestConnectionTimeouts.java @@ -33,6 +33,7 @@ import org.apache.hadoop.fs.FutureDataInputStreamBuilder; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.contract.ContractTestUtils; +import org.apache.hadoop.fs.s3a.AWSClientIOException; import org.apache.hadoop.fs.s3a.AbstractS3ATestBase; import org.apache.hadoop.fs.s3a.S3AFileSystem; import org.apache.hadoop.fs.s3a.api.PerformanceFlagEnum; @@ -59,8 +60,10 @@ import static org.apache.hadoop.fs.s3a.Constants.REQUEST_TIMEOUT; import static org.apache.hadoop.fs.s3a.Constants.RETRY_LIMIT; import static org.apache.hadoop.fs.s3a.Constants.SOCKET_TIMEOUT; +import static org.apache.hadoop.fs.s3a.S3ATestUtils.getTestBucketName; import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides; import static org.apache.hadoop.fs.s3a.commit.CommitConstants.MAGIC_PATH_PREFIX; +import static org.apache.hadoop.fs.s3a.S3AUtils.getEncryptionAlgorithm; import static org.apache.hadoop.fs.s3a.impl.ConfigurationHelper.setDurationAsMillis; import static org.apache.hadoop.test.LambdaTestUtils.intercept; @@ -156,7 +159,12 @@ public void testGeneratePoolTimeouts() throws Throwable { ContractTestUtils.createFile(fs, path, true, DATASET); final FileStatus st = fs.getFileStatus(path); try (FileSystem brittleFS = FileSystem.newInstance(fs.getUri(), conf)) { - intercept(ConnectTimeoutException.class, () -> { + Class exceptionClass = ConnectTimeoutException.class; + if (CSEUtils.isCSEEnabled(getEncryptionAlgorithm( + getTestBucketName(conf), conf).getMethod())) { + exceptionClass = AWSClientIOException.class; + } + intercept(exceptionClass, () -> { for (int i = 0; i < streamsToCreate; i++) { FutureDataInputStreamBuilder b = brittleFS.openFile(path); b.withFileStatus(st); diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestClientManager.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestClientManager.java index 857df58f42bb1c..a807cf6c4cbf0d 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestClientManager.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestClientManager.java @@ -119,6 +119,7 @@ private StubS3ClientFactory factory(final InvocationRaisingIOE invocationRaising private ClientManager manager(final StubS3ClientFactory factory) { return new ClientManagerImpl( factory, + null, new S3ClientFactory.S3ClientCreationParameters() .withPathUri(uri), StubDurationTrackerFactory.STUB_DURATION_TRACKER_FACTORY); diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestRequestFactory.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestRequestFactory.java index b864cd3b63982e..321f47f80ec270 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestRequestFactory.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestRequestFactory.java @@ -200,11 +200,13 @@ public void testMultipartUploadRequest() throws Throwable { String path = "path"; String id = "1"; - a(factory.newUploadPartRequestBuilder(path, id, 1, 0)); - a(factory.newUploadPartRequestBuilder(path, id, 2, 128_000_000)); + a(factory.newUploadPartRequestBuilder(path, id, 1, false, 0)); + a(factory.newUploadPartRequestBuilder(path, id, 2, false, + 128_000_000)); // partNumber is past the limit intercept(PathIOException.class, () -> - factory.newUploadPartRequestBuilder(path, id, 3, 128_000_000)); + factory.newUploadPartRequestBuilder(path, id, 3, true, + 128_000_000)); assertThat(countRequests.counter.get()) .describedAs("request preparation count") @@ -241,7 +243,9 @@ public void testDefaultUploadTimeouts() throws Throwable { .withMultipartPartCountLimit(2) .build(); final UploadPartRequest upload = - factory.newUploadPartRequestBuilder("path", "id", 2, 128_000_000).build(); + factory.newUploadPartRequestBuilder("path", "id", 2, + true, 128_000_000) + .build(); assertApiTimeouts(DEFAULT_PART_UPLOAD_TIMEOUT, upload); } @@ -266,7 +270,7 @@ public void testUploadTimeouts() throws Throwable { // multipart part final UploadPartRequest upload = factory.newUploadPartRequestBuilder(path, - "1", 3, 128_000_000) + "1", 3, false,128_000_000) .build(); assertApiTimeouts(partDuration, upload); diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/test/MinimalListingOperationCallbacks.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/test/MinimalListingOperationCallbacks.java index 2a8cfc663a53ac..e4d76faff7eb63 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/test/MinimalListingOperationCallbacks.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/test/MinimalListingOperationCallbacks.java @@ -30,6 +30,8 @@ import org.apache.hadoop.fs.statistics.DurationTrackerFactory; import org.apache.hadoop.fs.store.audit.AuditSpan; +import software.amazon.awssdk.services.s3.model.S3Object; + /** * Stub implementation of {@link ListingOperationCallbacks}. */ @@ -70,6 +72,12 @@ public long getDefaultBlockSize(Path path) { return 0; } + + @Override + public long getObjectSize(S3Object s3Object) throws IOException { + return 0; + } + @Override public int getMaxKeys() { return 0; From 1deb43a78d7ba5a6b11f2654cfe8092c2bf35faa Mon Sep 17 00:00:00 2001 From: Syed Shameerur Rahman Date: Thu, 31 Oct 2024 15:19:42 +0530 Subject: [PATCH 2/6] Addressing review comments --- .../org/apache/hadoop/fs/s3a/Constants.java | 15 +- .../hadoop/fs/s3a/DefaultS3ClientFactory.java | 2 +- .../org/apache/hadoop/fs/s3a/Listing.java | 180 +--------------- .../apache/hadoop/fs/s3a/S3AFileSystem.java | 71 ++----- .../org/apache/hadoop/fs/s3a/S3AStore.java | 38 ++++ .../apache/hadoop/fs/s3a/S3ClientFactory.java | 26 +++ .../fs/s3a/impl/BaseS3AFileSystemHandler.java | 39 +--- .../fs/s3a/impl/CSES3AFileSystemHandler.java | 53 +---- .../apache/hadoop/fs/s3a/impl/CSEUtils.java | 134 ++++++------ .../CSEV1CompatibleS3AFileSystemHandler.java | 47 +---- .../hadoop/fs/s3a/impl/ClientManager.java | 6 + .../s3a/impl/EncryptionS3ClientFactory.java | 7 +- .../fs/s3a/impl/S3AFileSystemHandler.java | 37 +--- .../hadoop/fs/s3a/impl/S3AStoreImpl.java | 106 ++++++++++ .../fs/s3a/prefetch/S3ARemoteObject.java | 2 +- .../markdown/tools/hadoop-aws/encryption.md | 33 ++- .../tools/hadoop-aws/troubleshooting_s3a.md | 65 ++---- .../apache/hadoop/fs/s3a/CustomKeyring.java | 6 +- .../fs/s3a/ITestS3AClientSideEncryption.java | 198 ++++++------------ .../ITestS3AClientSideEncryptionCustom.java | 2 +- .../fs/s3a/impl/ITestConnectionTimeouts.java | 1 + .../MinimalListingOperationCallbacks.java | 4 +- 22 files changed, 435 insertions(+), 637 deletions(-) diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java index 5d98494a278e81..4d52ee0d2f9c5e 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java @@ -777,11 +777,9 @@ private Constants() { /** * Config to provide backward compatibility with V1 encryption client. * Enabling this configuration will invoke the followings - * 1. Unencrypted s3 objects will be read using unecrypted/base s3 client when CSE is enabled. - * 2. Size of encrypted object will be calculated using ranged S3 calls. - * 3. While listing s3 objects, encryption metadata file with suffix - * {@link #S3_ENCRYPTION_CSE_INSTRUCTION_FILE_SUFFIX} will be skipped. - * This is to provide backward compatibility with V1 client. + * 1. Unencrypted s3 objects will be read using unencrypted/base s3 client when CSE is enabled. + * 2. Size of encrypted object will be fetched from object header if present or + * calculated using ranged S3 GET calls. * value:{@value} */ public static final String S3_ENCRYPTION_CSE_V1_COMPATIBILITY_ENABLED = @@ -793,12 +791,9 @@ private Constants() { public static final boolean S3_ENCRYPTION_CSE_V1_COMPATIBILITY_ENABLED_DEFAULT = false; /** - * Suffix of instruction file : {@value}. + * S3 CSE-KMS KMS region config. */ - public static final String S3_ENCRYPTION_CSE_INSTRUCTION_FILE_SUFFIX = ".instruction"; - - - + public static final String S3_ENCRYPTION_CSE_KMS_REGION = "fs.s3a.encryption.cse.kms.region"; /** * List of custom Signers. The signer class will be loaded, and the signer diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/DefaultS3ClientFactory.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/DefaultS3ClientFactory.java index 96a57ed087ee18..1e5a5648ecb42a 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/DefaultS3ClientFactory.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/DefaultS3ClientFactory.java @@ -380,7 +380,7 @@ private , ClientT> void * @param conf config to build the URI from. * @return an endpoint uri */ - public static URI getS3Endpoint(String endpoint, final Configuration conf) { + protected static URI getS3Endpoint(String endpoint, final Configuration conf) { boolean secureConnections = conf.getBoolean(SECURE_CONNECTIONS, DEFAULT_SECURE_CONNECTIONS); diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Listing.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Listing.java index c4319669d6527a..d8f68b48403398 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Listing.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Listing.java @@ -30,7 +30,6 @@ import org.apache.hadoop.fs.RemoteIterator; import org.apache.hadoop.fs.s3a.impl.AbstractStoreOperation; import org.apache.hadoop.fs.s3a.impl.ListingOperationCallbacks; -import org.apache.hadoop.fs.s3a.impl.S3AFileSystemHandler; import org.apache.hadoop.fs.s3a.impl.StoreContext; import org.apache.hadoop.fs.statistics.IOStatistics; import org.apache.hadoop.fs.statistics.IOStatisticsAggregator; @@ -59,7 +58,6 @@ import static org.apache.hadoop.fs.s3a.S3AUtils.objectRepresentsDirectory; import static org.apache.hadoop.fs.s3a.S3AUtils.stringify; import static org.apache.hadoop.fs.s3a.auth.RoleModel.pathToKey; -import static org.apache.hadoop.fs.s3a.impl.CSEUtils.isCSEInstructionFile; import static org.apache.hadoop.fs.statistics.StoreStatisticNames.OBJECT_CONTINUE_LIST_REQUEST; import static org.apache.hadoop.fs.statistics.StoreStatisticNames.OBJECT_LIST_REQUEST; import static org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding.iostatisticsStore; @@ -77,18 +75,16 @@ public class Listing extends AbstractStoreOperation { private static final Logger LOG = S3AFileSystem.LOG; - private final S3AFileSystemHandler handler; - public static final FileStatusAcceptor ACCEPT_ALL_BUT_S3N = + static final FileStatusAcceptor ACCEPT_ALL_BUT_S3N = new AcceptAllButS3nDirs(); private final ListingOperationCallbacks listingOperationCallbacks; public Listing(ListingOperationCallbacks listingOperationCallbacks, - StoreContext storeContext, S3AFileSystemHandler handler) { + StoreContext storeContext) { super(storeContext); this.listingOperationCallbacks = listingOperationCallbacks; - this.handler = handler; } /** @@ -239,7 +235,7 @@ public RemoteIterator getLocatedFileStatusIteratorForDir( listingOperationCallbacks .createListObjectsRequest(key, "/", span), filter, - handler.getFileStatusAcceptor(dir, false), + new AcceptAllButSelfAndS3nDirs(dir), span)); } @@ -268,7 +264,7 @@ public RemoteIterator getLocatedFileStatusIteratorForDir( path, request, ACCEPT_ALL, - handler.getFileStatusAcceptor(path, false), + new AcceptAllButSelfAndS3nDirs(path), span); } @@ -283,7 +279,7 @@ public S3ListRequest createListObjectsRequest(String key, * Interface to implement the logic deciding whether to accept a s3Object * entry or path as a valid file or directory. */ - public interface FileStatusAcceptor { + interface FileStatusAcceptor { /** * Predicate to decide whether or not to accept a s3Object entry. @@ -448,6 +444,7 @@ private boolean requestNextBatch() throws IOException { * Build the next status batch from a listing. * @param objects the next object listing * @return true if this added any entries after filtering + * @throws IOException IO problems. This can happen only when CSE is enabled. */ private boolean buildNextStatusBatch(S3ListResult objects) throws IOException { // counters for debug logs @@ -456,6 +453,8 @@ private boolean buildNextStatusBatch(S3ListResult objects) throws IOException { List stats = new ArrayList<>( objects.getS3Objects().size() + objects.getCommonPrefixes().size()); + String userName = getStoreContext().getUsername(); + long blockSize = listingOperationCallbacks.getDefaultBlockSize(null); // objects for (S3Object s3Object : objects.getS3Objects()) { String key = s3Object.key(); @@ -466,9 +465,8 @@ private boolean buildNextStatusBatch(S3ListResult objects) throws IOException { // Skip over keys that are ourselves and old S3N _$folder$ files if (acceptor.accept(keyPath, s3Object) && filter.accept(keyPath)) { S3AFileStatus status = createFileStatus(keyPath, s3Object, - listingOperationCallbacks.getDefaultBlockSize(keyPath), - getStoreContext().getUsername(), - s3Object.eTag(), null, + blockSize, userName, s3Object.eTag(), + null, listingOperationCallbacks.getObjectSize(s3Object)); LOG.debug("Adding: {}", status); stats.add(status); @@ -727,7 +725,7 @@ public void close() { * Accept all entries except the base path and those which map to S3N * pseudo directory markers. */ - public static class AcceptFilesOnly implements FileStatusAcceptor { + static class AcceptFilesOnly implements FileStatusAcceptor { private final Path qualifiedPath; public AcceptFilesOnly(Path qualifiedPath) { @@ -766,61 +764,6 @@ public boolean accept(FileStatus status) { } } - /** - * Accept all entries except the base path and those which map to S3N - * pseudo directory markers and CSE instruction file. - */ - public static class AcceptFilesOnlyExceptCSEInstructionFile implements FileStatusAcceptor { - private final Path qualifiedPath; - - /** - * Constructor. - * @param qualifiedPath an already-qualified path. - */ - public AcceptFilesOnlyExceptCSEInstructionFile(Path qualifiedPath) { - this.qualifiedPath = qualifiedPath; - } - - /** - * Reject a s3Object entry if the key path is the qualified Path, or - * it ends with {@code "_$folder$"} or {@code ".instruction"}. - * @param keyPath key path of the entry - * @param s3Object s3Object entry - * @return true if the entry is accepted (i.e. that a status entry - * should be generated. - */ - @Override - public boolean accept(Path keyPath, S3Object s3Object) { - return !keyPath.equals(qualifiedPath) - && !s3Object.key().endsWith(S3N_FOLDER_SUFFIX) - && !objectRepresentsDirectory(s3Object.key()) - && !isCSEInstructionFile(s3Object.key()); - } - - /** - * Accept no directory paths. - * @param keyPath qualified path to the entry - * @param prefix common prefix in listing. - * @return false, always. - */ - @Override - public boolean accept(Path keyPath, String prefix) { - return false; - } - - /** - * Reject if the file status is not a file or is a CSE instruction file. - * @param status file status containing file path information - * @return true if the entry is accepted (i.e. that a status entry - * should be generated. - */ - @Override - public boolean accept(FileStatus status) { - return (status != null) && status.isFile() - && !isCSEInstructionFile(status.getPath().toString()); - } - } - /** * Accept all entries except those which map to S3N pseudo directory markers. */ @@ -840,47 +783,6 @@ public boolean accept(FileStatus status) { } - /** - * Accept all entries except those which map to S3N pseudo directory markers and CSE instruction - * file. - */ - public static class AcceptAllButS3nDirsAndCSEInstructionFile implements FileStatusAcceptor { - - /** - * Reject a s3Object entry if the key ends with {@code "_$folder$"} or {@code ".instruction"}. - * @param keyPath key path of the entry - * @param s3Object s3Object entry - * @return true if the entry is accepted (i.e. that a status entry - * should be generated. - */ - public boolean accept(Path keyPath, S3Object s3Object) { - return !s3Object.key().endsWith(S3N_FOLDER_SUFFIX) && - !isCSEInstructionFile(s3Object.key()); - } - - /** - * Reject if the key ends with {@code "_$folder$"} or {@code ".instruction"}. - * @param keyPath qualified path to the entry - * @param prefix the prefix - * @return true if the entry is accepted - */ - public boolean accept(Path keyPath, String prefix) { - return !keyPath.toString().endsWith(S3N_FOLDER_SUFFIX) && - !isCSEInstructionFile(keyPath.toString()); - } - - /** - * Reject if the file status is a CSE instruction file or ends with {@code "_$folder$"}. - * @param status file status containing file path information - * @return true if the entry is accepted (i.e. that a status entry - * should be generated. - */ - public boolean accept(FileStatus status) { - return !status.getPath().toString().endsWith(S3N_FOLDER_SUFFIX) && - isCSEInstructionFile(status.getPath().toString()); - } - } - /** * Accept all entries except the base path and those which map to S3N * pseudo directory markers. @@ -930,66 +832,6 @@ public boolean accept(FileStatus status) { } } - /** - * Accept all entries except the base path, those which map to S3N pseudo directory markers - * and files which ends with CSE instruction file. - */ - public static class AcceptAllButSelfAndS3nDirsAndCSEInstructionFile - implements FileStatusAcceptor { - - /** Base path. */ - private final Path qualifiedPath; - - /** - * Constructor. - * @param qualifiedPath an already-qualified path. - */ - public AcceptAllButSelfAndS3nDirsAndCSEInstructionFile(Path qualifiedPath) { - this.qualifiedPath = qualifiedPath; - } - - /** - * Reject a s3Object entry if the key path is the qualified Path, or - * it ends with {@code "_$folder$"}. or ends with {@code ".instruction"}. - * @param keyPath key path of the entry - * @param s3Object s3Object entry - * @return true if the entry is accepted (i.e. that a status entry - * should be generated.) - */ - @Override - public boolean accept(Path keyPath, S3Object s3Object) { - return !keyPath.equals(qualifiedPath) && - !s3Object.key().endsWith(S3N_FOLDER_SUFFIX) && - !isCSEInstructionFile(s3Object.key()); - } - - /** - * Accept all prefixes except the one for the base path, "self" and CSE instruction file. - * @param keyPath qualified path to the entry - * @param prefix common prefix in listing. - * @return true if the entry is accepted (i.e. that a status entry - * should be generated. - */ - @Override - public boolean accept(Path keyPath, String prefix) { - return !keyPath.equals(qualifiedPath) && - !isCSEInstructionFile(keyPath.toString()); - } - - /** - * Reject if the file status is the base path, a CSE instruction file or ends - * with {@code "_$folder$"}. - * @param status file status containing file path information - * @return true if the entry is accepted (i.e. that a status entry - * should be generated. - */ - @Override - public boolean accept(FileStatus status) { - return (status != null) && !status.getPath().equals(qualifiedPath) && - !isCSEInstructionFile(status.getPath().toString()); - } - } - @SuppressWarnings("unchecked") public static RemoteIterator toLocatedFileStatusIterator( RemoteIterator iterator) { diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java index 1c81a3ff1b0330..0c5b5343df5ef1 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java @@ -74,7 +74,6 @@ import software.amazon.awssdk.services.s3.model.CopyObjectResponse; import software.amazon.awssdk.services.s3.model.DeleteObjectsRequest; import software.amazon.awssdk.services.s3.model.DeleteObjectsResponse; -import software.amazon.awssdk.services.s3.model.HeadObjectRequest; import software.amazon.awssdk.services.s3.model.HeadObjectResponse; import software.amazon.awssdk.services.s3.model.NoSuchBucketException; import software.amazon.awssdk.services.s3.model.ObjectIdentifier; @@ -248,7 +247,6 @@ import static org.apache.hadoop.fs.s3a.impl.CreateFileBuilder.OPTIONS_CREATE_FILE_NO_OVERWRITE; import static org.apache.hadoop.fs.s3a.impl.CreateFileBuilder.OPTIONS_CREATE_FILE_OVERWRITE; import static org.apache.hadoop.fs.s3a.impl.CreateFileBuilder.OPTIONS_CREATE_FILE_PERFORMANCE; -import static org.apache.hadoop.fs.s3a.impl.ErrorTranslation.isObjectNotFound; import static org.apache.hadoop.fs.s3a.impl.ErrorTranslation.isUnknownBucket; import static org.apache.hadoop.fs.s3a.impl.HeaderProcessing.CONTENT_TYPE_OCTET_STREAM; import static org.apache.hadoop.fs.s3a.impl.InternalConstants.AP_REQUIRED_EXCEPTION; @@ -782,7 +780,7 @@ public void initialize(URI name, Configuration originalConf) BULK_DELETE_PAGE_SIZE_DEFAULT, 0); checkArgument(pageSize <= InternalConstants.MAX_ENTRIES_TO_DELETE, "page size out of range: %s", pageSize); - listing = new Listing(listingOperationCallbacks, createStoreContext(), fsHandler); + listing = new Listing(listingOperationCallbacks, createStoreContext()); // now the open file logic openFileHelper = new OpenFileSupport( changeDetectionPolicy, @@ -1164,7 +1162,8 @@ private ClientManager createClientManager(URI fsURI, boolean dtEnabled) throws I .withChecksumValidationEnabled( conf.getBoolean(CHECKSUM_VALIDATION, CHECKSUM_VALIDATION_DEFAULT)) .withClientSideEncryptionEnabled(isCSEEnabled) - .withClientSideEncryptionMaterials(cseMaterials); + .withClientSideEncryptionMaterials(cseMaterials) + .withKMSRegion(conf.get(S3_ENCRYPTION_CSE_KMS_REGION)); // this is where clients and the transfer manager are created on demand. return createClientManager(clientFactory, unencryptedClientFactory, parameters, @@ -2646,8 +2645,14 @@ public RemoteIterator listFilesAndDirectoryMarkers( final S3AFileStatus status, final boolean includeSelf) throws IOException { auditSpan.activate(); - return innerListFiles(path, true, - fsHandler.getFileStatusAcceptor(path, includeSelf), status); + return innerListFiles( + path, + true, + includeSelf + ? Listing.ACCEPT_ALL_BUT_S3N + : new Listing.AcceptAllButSelfAndS3nDirs(path), + status + ); } @Override @@ -2693,7 +2698,7 @@ public RemoteIterator listObjects( listing.createFileStatusListingIterator(path, createListObjectsRequest(key, null), ACCEPT_ALL, - fsHandler.getFileStatusAcceptor(path, true), + Listing.ACCEPT_ALL_BUT_S3N, auditSpan)); } @@ -2798,8 +2803,7 @@ public long getDefaultBlockSize(Path path) { */ @Override public long getObjectSize(S3Object s3Object) throws IOException { - return fsHandler.getS3ObjectSize(s3Object.key(), s3Object.size(), store, bucket, - getRequestFactory(), null); + return fsHandler.getS3ObjectSize(s3Object.key(), s3Object.size(), store, null); } @Override @@ -3068,45 +3072,7 @@ protected HeadObjectResponse getObjectMetadata(String key, ChangeTracker changeTracker, Invoker changeInvoker, String operation) throws IOException { - HeadObjectResponse response = changeInvoker.retryUntranslated("GET " + key, true, - () -> { - HeadObjectRequest.Builder requestBuilder = - getRequestFactory().newHeadObjectRequestBuilder(key); - incrementStatistic(OBJECT_METADATA_REQUESTS); - DurationTracker duration = getDurationTrackerFactory() - .trackDuration(ACTION_HTTP_HEAD_REQUEST.getSymbol()); - try { - LOG.debug("HEAD {} with change tracker {}", key, changeTracker); - if (changeTracker != null) { - changeTracker.maybeApplyConstraint(requestBuilder); - } - HeadObjectResponse headObjectResponse = getS3Client().headObject( - requestBuilder.build()); - long length = fsHandler.getS3ObjectSize(key, headObjectResponse.contentLength(), - store, bucket, getRequestFactory(), headObjectResponse); - // overwrite the content length - headObjectResponse = headObjectResponse.toBuilder() - .contentLength(length) - .build(); - if (changeTracker != null) { - changeTracker.processMetadata(headObjectResponse, operation); - } - return headObjectResponse; - } catch (AwsServiceException ase) { - if (!isObjectNotFound(ase)) { - // file not found is not considered a failure of the call, - // so only switch the duration tracker to update failure - // metrics on other exception outcomes. - duration.failed(); - } - throw ase; - } finally { - // update the tracker. - duration.close(); - } - }); - incrementReadOperations(); - return response; + return store.headObject(key, changeTracker, changeInvoker, fsHandler, operation); } /** @@ -3762,7 +3728,7 @@ private RemoteIterator innerListStatus(Path f) return listing.createProvidedFileStatusIterator( stats, ACCEPT_ALL, - fsHandler.getFileStatusAcceptor(path, true)); + Listing.ACCEPT_ALL_BUT_S3N); } } // Here we have a directory which may or may not be empty. @@ -3960,8 +3926,7 @@ public S3AFileStatus probePathStatus(final Path path, @Override public RemoteIterator listFilesIterator(final Path path, final boolean recursive) throws IOException { - return S3AFileSystem.this.innerListFiles(path, recursive, - fsHandler.getFileStatusAcceptor(path, true), null); + return S3AFileSystem.this.innerListFiles(path, recursive, Listing.ACCEPT_ALL_BUT_S3N, null); } } @@ -5253,7 +5218,7 @@ public RemoteIterator listFiles(Path f, return toLocatedFileStatusIterator( trackDurationAndSpan(INVOCATION_LIST_FILES, path, () -> innerListFiles(path, recursive, - fsHandler.getFileStatusAcceptor(path), null))); + new Listing.AcceptFilesOnly(path), null))); } /** @@ -5271,7 +5236,7 @@ public RemoteIterator listFilesAndEmptyDirectories( final Path path = qualify(f); return trackDurationAndSpan(INVOCATION_LIST_FILES, path, () -> innerListFiles(path, recursive, - fsHandler.getFileStatusAcceptor(path, true), + Listing.ACCEPT_ALL_BUT_S3N, null)); } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AStore.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AStore.java index aed44427169632..b10769dfe73f39 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AStore.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AStore.java @@ -27,6 +27,7 @@ import java.util.concurrent.CancellationException; import software.amazon.awssdk.awscore.exception.AwsServiceException; +import software.amazon.awssdk.core.ResponseInputStream; import software.amazon.awssdk.core.exception.SdkException; import software.amazon.awssdk.core.sync.RequestBody; import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadRequest; @@ -35,6 +36,8 @@ import software.amazon.awssdk.services.s3.model.DeleteObjectResponse; import software.amazon.awssdk.services.s3.model.DeleteObjectsRequest; import software.amazon.awssdk.services.s3.model.DeleteObjectsResponse; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; +import software.amazon.awssdk.services.s3.model.HeadObjectResponse; import software.amazon.awssdk.services.s3.model.PutObjectRequest; import software.amazon.awssdk.services.s3.model.UploadPartRequest; import software.amazon.awssdk.services.s3.model.UploadPartResponse; @@ -43,8 +46,10 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.fs.s3a.api.RequestFactory; +import org.apache.hadoop.fs.s3a.impl.ChangeTracker; import org.apache.hadoop.fs.s3a.impl.ClientManager; import org.apache.hadoop.fs.s3a.impl.MultiObjectDeleteException; +import org.apache.hadoop.fs.s3a.impl.S3AFileSystemHandler; import org.apache.hadoop.fs.s3a.impl.StoreContext; import org.apache.hadoop.fs.s3a.statistics.S3AStatisticsContext; import org.apache.hadoop.fs.statistics.DurationTrackerFactory; @@ -193,6 +198,39 @@ Map.Entry deleteObjects(DeleteObjectsRequest de Map.Entry> deleteObject( DeleteObjectRequest request) throws SdkException; + /** + * Performs a HEAD request on an S3 object to retrieve its metadata. + * + * @param key The S3 object key to perform the HEAD operation on + * @param changeTracker Tracks changes to the object's metadata across operations + * @param changeInvoker The invoker responsible for executing the HEAD request with retries + * @param fsHandler Handler for filesystem-level operations and configurations + * @param operation Description of the operation being performed for tracking purposes + * @return HeadObjectResponse containing the object's metadata + * @throws IOException If the HEAD request fails, object doesn't exist, or other I/O errors occur + */ + @Retries.RetryRaw + HeadObjectResponse headObject(String key, + ChangeTracker changeTracker, + Invoker changeInvoker, + S3AFileSystemHandler fsHandler, + String operation) throws IOException; + + /** + * Retrieves a specific byte range of an S3 object as a stream. + * + * @param key The S3 object key to retrieve + * @param start The starting byte position (inclusive) of the range to retrieve + * @param end The ending byte position (inclusive) of the range to retrieve + * @return A ResponseInputStream containing the requested byte range of the S3 object + * @throws IOException If the object cannot be retrieved other I/O errors occur + * @see GetObjectResponse For additional metadata about the retrieved object + */ + @Retries.RetryRaw + ResponseInputStream getRangedS3Object(String key, + long start, + long end) throws IOException; + /** * Upload part of a multi-partition file. * Increments the write and put counters. diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java index c18b7a1dcdd8f6..cda09b622eaff4 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java @@ -125,6 +125,12 @@ final class S3ClientCreationParameters { */ private Boolean isCSEEnabled = false; + /** + * KMS region. + * This is only used if CSE is enabled. + */ + private String kmsRegion; + /** * Client side encryption materials. */ @@ -451,6 +457,18 @@ public S3ClientCreationParameters withClientSideEncryptionEnabled(final boolean return this; } + /** + * Set the KMS client region. + * This is required for CSE-KMS + * + * @param value new value + * @return the builder + */ + public S3ClientCreationParameters withKMSRegion(final String value) { + this.kmsRegion = value; + return this; + } + /** * Get the client side encryption flag. * @return client side encryption flag @@ -486,6 +504,14 @@ public String getRegion() { return region; } + /** + * Get the KMS region. + * @return Configured KMS region. + */ + public String getKmsRegion() { + return kmsRegion; + } + /** * Should s3express createSession be called? * @return true if the client should enable createSession. diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/BaseS3AFileSystemHandler.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/BaseS3AFileSystemHandler.java index a0b3e75d55af17..2fc7447693db39 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/BaseS3AFileSystemHandler.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/BaseS3AFileSystemHandler.java @@ -21,8 +21,6 @@ import java.io.IOException; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.s3a.Listing; import org.apache.hadoop.fs.s3a.S3AEncryptionMethods; import org.apache.hadoop.fs.s3a.S3AStore; import org.apache.hadoop.fs.s3a.S3ClientFactory; @@ -51,34 +49,6 @@ public class BaseS3AFileSystemHandler implements S3AFileSystemHandler { public BaseS3AFileSystemHandler() { } - /** - * Returns a {@link Listing.FileStatusAcceptor} object. - * That determines which files and directories should be included in a listing operation. - * - * @param path the path for which the listing is being performed - * @param includeSelf a boolean indicating whether the path itself should - * be included in the listing - * @return a {@link Listing.FileStatusAcceptor} object - */ - @Override - public Listing.FileStatusAcceptor getFileStatusAcceptor(Path path, boolean includeSelf) { - return includeSelf - ? Listing.ACCEPT_ALL_BUT_S3N - : new Listing.AcceptAllButSelfAndS3nDirs(path); - } - - /** - * Returns a {@link Listing.FileStatusAcceptor} object. - * That determines which files and directories should be included in a listing operation. - * - * @param path the path for which the listing is being performed - * @return a {@link Listing.FileStatusAcceptor} object - */ - @Override - public Listing.FileStatusAcceptor getFileStatusAcceptor(Path path) { - return new Listing.AcceptFilesOnly(path); - } - /** * Retrieves an object from the S3. * @@ -89,7 +59,8 @@ public Listing.FileStatusAcceptor getFileStatusAcceptor(Path path) { * @throws IOException If an error occurs while retrieving the object. */ @Override - public ResponseInputStream getObject(S3AStore store, GetObjectRequest request, + public ResponseInputStream getObject(S3AStore store, + GetObjectRequest request, RequestFactory factory) throws IOException { return store.getOrCreateS3Client().getObject(request); } @@ -149,14 +120,12 @@ public S3ClientFactory getUnencryptedS3ClientFactory(Configuration conf) { * @param key The key (path) of the object in the S3 bucket. * @param length The expected length of the object. * @param store The S3AStore object representing the S3 bucket. - * @param bucket The name of the S3 bucket. - * @param factory The RequestFactory used to create the HeadObjectRequest. * @param response The HeadObjectResponse containing the metadata of the object. * @return The size of the object in bytes. */ @Override - public long getS3ObjectSize(String key, long length, S3AStore store, String bucket, - RequestFactory factory, HeadObjectResponse response) { + public long getS3ObjectSize(String key, long length, S3AStore store, + HeadObjectResponse response) throws IOException { return length; } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/CSES3AFileSystemHandler.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/CSES3AFileSystemHandler.java index 66a76c1028d23b..8529d7b180c682 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/CSES3AFileSystemHandler.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/CSES3AFileSystemHandler.java @@ -26,8 +26,6 @@ import software.amazon.awssdk.services.s3.model.HeadObjectResponse; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.s3a.Listing; import org.apache.hadoop.fs.s3a.S3AEncryptionMethods; import org.apache.hadoop.fs.s3a.S3AStore; import org.apache.hadoop.fs.s3a.S3ClientFactory; @@ -50,34 +48,6 @@ public class CSES3AFileSystemHandler implements S3AFileSystemHandler{ public CSES3AFileSystemHandler() { } - /** - * Returns a {@link Listing.FileStatusAcceptor} object. - * That determines which files and directories should be included in a listing operation. - * - * @param path the path for which the listing is being performed - * @param includeSelf a boolean indicating whether the path itself should - * be included in the listing - * @return a {@link Listing.FileStatusAcceptor} object - */ - @Override - public Listing.FileStatusAcceptor getFileStatusAcceptor(Path path, boolean includeSelf) { - return includeSelf - ? Listing.ACCEPT_ALL_BUT_S3N - : new Listing.AcceptAllButSelfAndS3nDirs(path); - } - - /** - * Returns a {@link Listing.FileStatusAcceptor} object. - * That determines which files and directories should be included in a listing operation. - * - * @param path the path for which the listing is being performed - * @return a {@link Listing.FileStatusAcceptor} object - */ - @Override - public Listing.FileStatusAcceptor getFileStatusAcceptor(Path path) { - return new Listing.AcceptFilesOnly(path); - } - /** * Retrieves an object from the S3 using encrypted S3 client. * @@ -88,7 +58,8 @@ public Listing.FileStatusAcceptor getFileStatusAcceptor(Path path) { * @throws IOException If an error occurs while retrieving the object. */ @Override - public ResponseInputStream getObject(S3AStore store, GetObjectRequest request, + public ResponseInputStream getObject(S3AStore store, + GetObjectRequest request, RequestFactory factory) throws IOException { return store.getOrCreateS3Client().getObject(request); } @@ -103,7 +74,7 @@ public void setCSEGauge(IOStatisticsStore ioStatisticsStore) { } /** - * Retrieves the client-side encryption materials for the given bucket and encryption algorithm. + * Retrieves the cse materials. * * @param conf The Hadoop configuration object. * @param bucket The name of the S3 bucket. @@ -141,23 +112,21 @@ public S3ClientFactory getUnencryptedS3ClientFactory(Configuration conf) { /** - * Retrieves the unpadded size of an object in the S3. + * Retrieves the unencrypted length of an object in the S3 bucket. * * @param key The key (path) of the object in the S3 bucket. - * @param length The expected length of the object, including any padding. + * @param length The length of the object. * @param store The S3AStore object representing the S3 bucket. - * @param bucket The name of the S3 bucket. - * @param factory The RequestFactory used to create the HeadObjectRequest. * @param response The HeadObjectResponse containing the metadata of the object. - * @return The unpadded size of the object in bytes. + * @return The unencrypted size of the object in bytes. * @throws IOException If an error occurs while retrieving the object size. */ @Override - public long getS3ObjectSize(String key, long length, S3AStore store, String bucket, - RequestFactory factory, HeadObjectResponse response) throws IOException { - long unpaddedLength = length - CSE_PADDING_LENGTH; - if (unpaddedLength >= 0) { - return unpaddedLength; + public long getS3ObjectSize(String key, long length, S3AStore store, + HeadObjectResponse response) throws IOException { + long unencryptedLength = length - CSE_PADDING_LENGTH; + if (unencryptedLength >= 0) { + return unencryptedLength; } return length; } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/CSEUtils.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/CSEUtils.java index 100fdc0ec54380..ddb0040cbcb58f 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/CSEUtils.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/CSEUtils.java @@ -25,20 +25,14 @@ import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.s3a.S3AEncryptionMethods; -import org.apache.hadoop.fs.s3a.api.RequestFactory; +import org.apache.hadoop.fs.s3a.S3AStore; import org.apache.hadoop.util.Preconditions; -import software.amazon.awssdk.services.s3.S3Client; -import software.amazon.awssdk.services.s3.model.GetObjectRequest; -import software.amazon.awssdk.services.s3.model.HeadObjectRequest; import software.amazon.awssdk.services.s3.model.HeadObjectResponse; -import software.amazon.awssdk.services.s3.model.NoSuchKeyException; import static org.apache.hadoop.fs.s3a.Constants.S3_ENCRYPTION_CSE_CUSTOM_KEYRING_CLASS_NAME; -import static org.apache.hadoop.fs.s3a.Constants.S3_ENCRYPTION_CSE_INSTRUCTION_FILE_SUFFIX; import static org.apache.hadoop.fs.s3a.S3AEncryptionMethods.CSE_CUSTOM; import static org.apache.hadoop.fs.s3a.S3AEncryptionMethods.CSE_KMS; -import static org.apache.hadoop.fs.s3a.S3AUtils.formatRange; import static org.apache.hadoop.fs.s3a.S3AUtils.getS3EncryptionKey; import static org.apache.hadoop.fs.s3a.impl.AWSHeaders.CRYPTO_CEK_ALGORITHM; import static org.apache.hadoop.fs.s3a.impl.AWSHeaders.UNENCRYPTED_CONTENT_LENGTH; @@ -47,7 +41,7 @@ /** * S3 client side encryption (CSE) utility class. */ -@InterfaceAudience.Public +@InterfaceAudience.Private @InterfaceStability.Evolving public final class CSEUtils { @@ -55,20 +49,16 @@ private CSEUtils() { } /** - * Checks if the file suffix ends CSE file suffix. - * {@link org.apache.hadoop.fs.s3a.Constants#S3_ENCRYPTION_CSE_INSTRUCTION_FILE_SUFFIX} - * when the config - * @param key file name - * @return true if file name ends with CSE instruction file suffix - */ - public static boolean isCSEInstructionFile(String key) { - return key.endsWith(S3_ENCRYPTION_CSE_INSTRUCTION_FILE_SUFFIX); - } - - /** - * Checks if CSE-KMS or CSE-CUSTOM is set. - * @param encryptionMethod type of encryption used - * @return true if encryption method is CSE-KMS or CSE-CUSTOM + * Checks if Client-Side Encryption (CSE) is enabled based on the encryption method. + * + * Validates if the provided encryption method matches either CSE-KMS or CSE-Custom + * encryption methods. These are the two supported client-side encryption methods. + * + * @param encryptionMethod The encryption method to check (case-sensitive) + * @return true if the encryption method is either CSE-KMS or CSE-Custom, + * false otherwise + * @see S3AEncryptionMethods#CSE_KMS + * @see S3AEncryptionMethods#CSE_CUSTOM */ public static boolean isCSEEnabled(String encryptionMethod) { return CSE_KMS.getMethod().equals(encryptionMethod) || @@ -76,54 +66,59 @@ public static boolean isCSEEnabled(String encryptionMethod) { } /** - * Checks if a given S3 object is encrypted or not by checking following two conditions - * 1. if object metadata contains x-amz-cek-alg - * 2. if instruction file is present + * Checks if an S3 object is encrypted by examining its metadata. * - * @param s3Client S3 client - * @param factory S3 request factory - * @param key key value of the s3 object - * @return true if S3 object is encrypted + * This method performs a HEAD request on the object and checks for the presence + * of encryption metadata (specifically the CEK algorithm indicator). + * + * @param store The S3AStore instance used to access the S3 object + * @param key The key (path) of the S3 object to check + * @return true if the object is encrypted (has CEK algorithm metadata), + * false otherwise + * @throws IOException If there's an error accessing the object metadata or + * communicating with S3 */ - public static boolean isObjectEncrypted(S3Client s3Client, RequestFactory factory, String key) { - HeadObjectRequest.Builder requestBuilder = factory.newHeadObjectRequestBuilder(key); - HeadObjectResponse headObjectResponse = s3Client.headObject(requestBuilder.build()); + public static boolean isObjectEncrypted(S3AStore store, + String key) throws IOException { + HeadObjectResponse headObjectResponse = store.headObject(key, + null, + null, + null, + "getObjectMetadata"); + if (headObjectResponse.hasMetadata() && headObjectResponse.metadata().get(CRYPTO_CEK_ALGORITHM) != null) { return true; } - HeadObjectRequest.Builder instructionFileRequestBuilder = - factory.newHeadObjectRequestBuilder(key + S3_ENCRYPTION_CSE_INSTRUCTION_FILE_SUFFIX); - try { - s3Client.headObject(instructionFileRequestBuilder.build()); - return true; - } catch (NoSuchKeyException e) { - // Ignore. This indicates no instruction file is present - } return false; } /** - * Get the unencrypted length of the object. + * Determines the actual unencrypted length of an S3 object. * - * @param s3Client S3 client - * @param bucket bucket name of the s3 object - * @param key key value of the s3 object - * @param factory S3 request factory - * @param contentLength S3 object length - * @param headObjectResponse response from headObject call - * @return unencrypted length of the object - * @throws IOException IO failures + * This method uses a three-step process to determine the object's unencrypted length: + * 1. If the object is not encrypted, returns the original content length + * 2. If encrypted, attempts to read the unencrypted length from object metadata + * 3. If metadata is unavailable, calculates length by performing a ranged GET operation + * + * @param store The S3AStore instance used to access the S3 object + * @param key The key (path) of the S3 object + * @param contentLength The encrypted object's content length + * @param headObjectResponse The object's metadata from a HEAD request, may be null + * @return The length of the object's unencrypted content + * @throws IOException If there's an error: + * - accessing the object or its metadata + * - parsing the unencrypted length from metadata + * - performing the ranged GET operation + * - computing the unencrypted length */ - public static long getUnPaddedObjectLength(S3Client s3Client, - String bucket, + public static long getUnencryptedObjectLength(S3AStore store, String key, - RequestFactory factory, long contentLength, HeadObjectResponse headObjectResponse) throws IOException { // if object is unencrypted, return the actual size - if (!isObjectEncrypted(s3Client, factory, key)) { + if (!isObjectEncrypted(store, key)) { return contentLength; } @@ -142,12 +137,7 @@ public static long getUnPaddedObjectLength(S3Client s3Client, if (minPlaintextLength < 0) { minPlaintextLength = 0; } - GetObjectRequest getObjectRequest = GetObjectRequest.builder() - .bucket(bucket) - .key(key) - .range(formatRange(minPlaintextLength, contentLength)) - .build(); - try (InputStream is = s3Client.getObject(getObjectRequest)) { + try (InputStream is = store.getRangedS3Object(key, minPlaintextLength, contentLength)) { int i = 0; while (is.read() != -1) { i++; @@ -161,14 +151,26 @@ public static long getUnPaddedObjectLength(S3Client s3Client, } /** - * Configure CSE params based on encryption algorithm. - * @param conf Configuration - * @param bucket bucket name - * @param algorithm encryption algorithm - * @return CSEMaterials - * @throws IOException IO failures + * Creates encryption materials for client-side encryption based on the specified algorithm. + * + * Supports two types of client-side encryption: + *

    + *
  • CSE_KMS: Uses AWS KMS for key management
    • + *
  • CSE_CUSTOM: Uses a custom cryptographic implementation
    • + *
+ * + * @param conf The configuration containing encryption settings + * @param bucket The S3 bucket name for which encryption materials are being created + * @param algorithm The encryption algorithm to use (CSE_KMS or CSE_CUSTOM) + * @return CSEMaterials configured with the appropriate encryption settings + * @throws IOException If there's an error retrieving encryption configuration + * @throws IllegalArgumentException If: + * - KMS key ID is null or empty (for CSE_KMS) + * - Custom crypto class name is null or empty (for CSE_CUSTOM) + * - Unsupported encryption algorithm is specified */ - public static CSEMaterials getClientSideEncryptionMaterials(Configuration conf, String bucket, + public static CSEMaterials getClientSideEncryptionMaterials(Configuration conf, + String bucket, S3AEncryptionMethods algorithm) throws IOException { switch (algorithm) { case CSE_KMS: diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/CSEV1CompatibleS3AFileSystemHandler.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/CSEV1CompatibleS3AFileSystemHandler.java index 25b09a4680d056..493357d4e8ccdb 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/CSEV1CompatibleS3AFileSystemHandler.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/CSEV1CompatibleS3AFileSystemHandler.java @@ -21,8 +21,6 @@ import java.io.IOException; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.s3a.Listing; import org.apache.hadoop.fs.s3a.S3AStore; import org.apache.hadoop.fs.s3a.S3ClientFactory; import org.apache.hadoop.fs.s3a.api.RequestFactory; @@ -51,32 +49,6 @@ public class CSEV1CompatibleS3AFileSystemHandler extends CSES3AFileSystemHandler public CSEV1CompatibleS3AFileSystemHandler() { } - /** - * {@inheritDoc} - * - *

This implementation returns a {@link Listing.AcceptAllButS3nDirsAndCSEInstructionFile} - * or {@link Listing.AcceptAllButSelfAndS3nDirsAndCSEInstructionFile} object - * based on the value of the {@code includeSelf} parameter. - */ - @Override - public Listing.FileStatusAcceptor getFileStatusAcceptor(Path path, boolean includeSelf) { - return includeSelf - ? new Listing.AcceptAllButS3nDirsAndCSEInstructionFile() - : new Listing.AcceptAllButSelfAndS3nDirsAndCSEInstructionFile(path); - } - - /** - * Returns a {@link Listing.FileStatusAcceptor} object. - * That determines which files and directories should be included in a listing operation. - * - * @param path the path for which the listing is being performed - * @return a {@link Listing.FileStatusAcceptor} object - */ - @Override - public Listing.FileStatusAcceptor getFileStatusAcceptor(Path path) { - return new Listing.AcceptFilesOnlyExceptCSEInstructionFile(path); - } - /** * Retrieves an object from the S3. * If the S3 object is encrypted, it uses the encrypted S3 client to retrieve the object else @@ -89,9 +61,10 @@ public Listing.FileStatusAcceptor getFileStatusAcceptor(Path path) { * @throws IOException If an error occurs while retrieving the object. */ @Override - public ResponseInputStream getObject(S3AStore store, GetObjectRequest request, + public ResponseInputStream getObject(S3AStore store, + GetObjectRequest request, RequestFactory factory) throws IOException { - boolean isEncrypted = isObjectEncrypted(store.getOrCreateS3Client(), factory, request.key()); + boolean isEncrypted = isObjectEncrypted(store, request.key()); return isEncrypted ? store.getOrCreateS3Client().getObject(request) : store.getOrCreateUnencryptedS3Client().getObject(request); } @@ -110,23 +83,19 @@ public S3ClientFactory getUnencryptedS3ClientFactory(Configuration conf) { return ReflectionUtils.newInstance(s3ClientFactoryClass, conf); } - /** - * Retrieves the unpadded size of an object in the S3 bucket. + * Retrieves the unencrypted length of an object in the S3 bucket. * * @param key The key (path) of the object in the S3 bucket. * @param length The length of the object. * @param store The S3AStore object representing the S3 bucket. - * @param bucket The name of the S3 bucket. - * @param factory The RequestFactory used to create the HeadObjectRequest. * @param response The HeadObjectResponse containing the metadata of the object. - * @return The unpadded size of the object in bytes. + * @return The unencrypted size of the object in bytes. * @throws IOException If an error occurs while retrieving the object size. */ @Override - public long getS3ObjectSize(String key, long length, S3AStore store, String bucket, - RequestFactory factory, HeadObjectResponse response) throws IOException { - return CSEUtils.getUnPaddedObjectLength(store.getOrCreateS3Client(), bucket, - key, factory, length, response); + public long getS3ObjectSize(String key, long length, S3AStore store, + HeadObjectResponse response) throws IOException { + return CSEUtils.getUnencryptedObjectLength(store, key, length, response); } } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/ClientManager.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/ClientManager.java index c9eaeebae4b1b3..ad7afc732387ff 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/ClientManager.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/ClientManager.java @@ -61,6 +61,12 @@ S3TransferManager getOrCreateTransferManager() */ S3AsyncClient getOrCreateAsyncClient() throws IOException; + /** + * Get or create an unencrypted S3 client. + * This is used for unencrypted operations when CSE is enabled with V1 compatibility. + * @return unencrypted S3 client + * @throws IOException on any failure + */ S3Client getOrCreateUnencryptedS3Client() throws IOException; /** diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/EncryptionS3ClientFactory.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/EncryptionS3ClientFactory.java index e89bb71445f58c..57f1173dc63749 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/EncryptionS3ClientFactory.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/EncryptionS3ClientFactory.java @@ -197,9 +197,14 @@ private Keyring createKmsKeyring(S3ClientCreationParameters parameters, if (parameters.getCredentialSet() != null) { kmsClientBuilder.credentialsProvider(parameters.getCredentialSet()); } - if (parameters.getRegion() != null) { + // check if kms region is configured. + if (parameters.getKmsRegion() != null) { + kmsClientBuilder.region(Region.of(parameters.getKmsRegion())); + } else if (parameters.getRegion() != null) { + // fallback to s3 region if kms region is not configured. kmsClientBuilder.region(Region.of(parameters.getRegion())); } else if (parameters.getEndpoint() != null) { + // fallback to s3 endpoint config if both kms region and s3 region is not set. String endpointStr = parameters.getEndpoint(); URI endpoint = getS3Endpoint(endpointStr, cseMaterials.getConf()); kmsClientBuilder.endpointOverride(endpoint); diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/S3AFileSystemHandler.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/S3AFileSystemHandler.java index 734bbcca07933c..00150ea03fd853 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/S3AFileSystemHandler.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/S3AFileSystemHandler.java @@ -26,8 +26,6 @@ import software.amazon.awssdk.services.s3.model.HeadObjectResponse; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.s3a.Listing; import org.apache.hadoop.fs.s3a.S3AEncryptionMethods; import org.apache.hadoop.fs.s3a.S3AStore; import org.apache.hadoop.fs.s3a.S3ClientFactory; @@ -35,30 +33,11 @@ import org.apache.hadoop.fs.statistics.impl.IOStatisticsStore; /** - * An interface that defines the contract for handling certain filesystem operations. + * An interface that helps map from object store semantics to that of the fileystem. + * This specially supports encrypted stores. */ public interface S3AFileSystemHandler { - /** - * Returns a {@link Listing.FileStatusAcceptor} object. - * That determines which files and directories should be included in a listing operation. - * - * @param path the path for which the listing is being performed - * @param includeSelf a boolean indicating whether the path itself should - * be included in the listing - * @return a {@link Listing.FileStatusAcceptor} object - */ - Listing.FileStatusAcceptor getFileStatusAcceptor(Path path, boolean includeSelf); - - /** - * Returns a {@link Listing.FileStatusAcceptor} object. - * That determines which files and directories should be included in a listing operation. - * - * @param path the path for which the listing is being performed - * @return a {@link Listing.FileStatusAcceptor} object - */ - Listing.FileStatusAcceptor getFileStatusAcceptor(Path path); - /** * Retrieves an object from the S3. * @@ -107,19 +86,17 @@ CSEMaterials getClientSideEncryptionMaterials(Configuration conf, String bucket, /** - * Retrieves the size of a S3 object. + * Retrieves the unencrypted length of an object in the S3 bucket. * * @param key The key (path) of the object in the S3 bucket. - * @param length The expected length of the object, if known. If not known, pass -1. + * @param length The length of the object. * @param store The S3AStore object representing the S3 bucket. - * @param bucket The name of the S3 bucket. - * @param factory The RequestFactory used to create the HeadObjectRequest. * @param response The HeadObjectResponse containing the metadata of the object. - * @return The size of the object in bytes. + * @return The unencrypted size of the object in bytes. * @throws IOException If an error occurs while retrieving the object size. */ - long getS3ObjectSize(String key, long length, S3AStore store, String bucket, - RequestFactory factory, HeadObjectResponse response) throws IOException; + long getS3ObjectSize(String key, long length, S3AStore store, + HeadObjectResponse response) throws IOException; } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/S3AStoreImpl.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/S3AStoreImpl.java index 38fb5a1e7f6424..04325634fa3304 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/S3AStoreImpl.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/S3AStoreImpl.java @@ -32,6 +32,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import software.amazon.awssdk.awscore.exception.AwsServiceException; +import software.amazon.awssdk.core.ResponseInputStream; import software.amazon.awssdk.core.exception.SdkException; import software.amazon.awssdk.core.sync.RequestBody; import software.amazon.awssdk.services.s3.S3AsyncClient; @@ -42,6 +43,10 @@ import software.amazon.awssdk.services.s3.model.DeleteObjectResponse; import software.amazon.awssdk.services.s3.model.DeleteObjectsRequest; import software.amazon.awssdk.services.s3.model.DeleteObjectsResponse; +import software.amazon.awssdk.services.s3.model.GetObjectRequest; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; +import software.amazon.awssdk.services.s3.model.HeadObjectRequest; +import software.amazon.awssdk.services.s3.model.HeadObjectResponse; import software.amazon.awssdk.services.s3.model.ObjectIdentifier; import software.amazon.awssdk.services.s3.model.PutObjectRequest; import software.amazon.awssdk.services.s3.model.S3Error; @@ -59,11 +64,13 @@ import org.apache.hadoop.fs.s3a.S3AInstrumentation; import org.apache.hadoop.fs.s3a.S3AStorageStatistics; import org.apache.hadoop.fs.s3a.S3AStore; +import org.apache.hadoop.fs.s3a.S3AUtils; import org.apache.hadoop.fs.s3a.Statistic; import org.apache.hadoop.fs.s3a.UploadInfo; import org.apache.hadoop.fs.s3a.api.RequestFactory; import org.apache.hadoop.fs.s3a.audit.AuditSpanS3A; import org.apache.hadoop.fs.s3a.statistics.S3AStatisticsContext; +import org.apache.hadoop.fs.statistics.DurationTracker; import org.apache.hadoop.fs.statistics.DurationTrackerFactory; import org.apache.hadoop.fs.statistics.IOStatistics; import org.apache.hadoop.fs.store.audit.AuditSpanSource; @@ -75,11 +82,13 @@ import static org.apache.hadoop.fs.s3a.S3AUtils.extractException; import static org.apache.hadoop.fs.s3a.S3AUtils.getPutRequestLength; import static org.apache.hadoop.fs.s3a.S3AUtils.isThrottleException; +import static org.apache.hadoop.fs.s3a.Statistic.ACTION_HTTP_HEAD_REQUEST; import static org.apache.hadoop.fs.s3a.Statistic.IGNORED_ERRORS; import static org.apache.hadoop.fs.s3a.Statistic.MULTIPART_UPLOAD_PART_PUT; import static org.apache.hadoop.fs.s3a.Statistic.OBJECT_BULK_DELETE_REQUEST; import static org.apache.hadoop.fs.s3a.Statistic.OBJECT_DELETE_OBJECTS; import static org.apache.hadoop.fs.s3a.Statistic.OBJECT_DELETE_REQUEST; +import static org.apache.hadoop.fs.s3a.Statistic.OBJECT_METADATA_REQUESTS; import static org.apache.hadoop.fs.s3a.Statistic.OBJECT_PUT_BYTES; import static org.apache.hadoop.fs.s3a.Statistic.OBJECT_PUT_BYTES_PENDING; import static org.apache.hadoop.fs.s3a.Statistic.OBJECT_PUT_REQUESTS_ACTIVE; @@ -90,6 +99,7 @@ import static org.apache.hadoop.fs.s3a.Statistic.STORE_IO_THROTTLE_RATE; import static org.apache.hadoop.fs.s3a.impl.ErrorTranslation.isObjectNotFound; import static org.apache.hadoop.fs.s3a.impl.InternalConstants.DELETE_CONSIDERED_IDEMPOTENT; +import static org.apache.hadoop.fs.statistics.StoreStatisticNames.ACTION_HTTP_GET_REQUEST; import static org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding.trackDurationOfOperation; import static org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding.trackDurationOfSupplier; import static org.apache.hadoop.util.Preconditions.checkArgument; @@ -545,6 +555,102 @@ public Map.Entry deleteObjects( } } + /** + * Performs a HEAD request on an S3 object to retrieve its metadata. + * + * @param key The S3 object key to perform the HEAD operation on + * @param changeTracker Tracks changes to the object's metadata across operations + * @param changeInvoker The invoker responsible for executing the HEAD request with retries + * @param fsHandler Handler for filesystem-level operations and configurations + * @param operation Description of the operation being performed for tracking purposes + * @return HeadObjectResponse containing the object's metadata + * @throws IOException If the HEAD request fails, object doesn't exist, or other I/O errors occur + */ + @Override + @Retries.RetryRaw + public HeadObjectResponse headObject(String key, + ChangeTracker changeTracker, + Invoker changeInvoker, + S3AFileSystemHandler fsHandler, + String operation) throws IOException { + HeadObjectResponse response = getStoreContext().getInvoker() + .retryUntranslated("GET " + key, true, + () -> { + HeadObjectRequest.Builder requestBuilder = + getRequestFactory().newHeadObjectRequestBuilder(key); + incrementStatistic(OBJECT_METADATA_REQUESTS); + DurationTracker duration = + getDurationTrackerFactory().trackDuration(ACTION_HTTP_HEAD_REQUEST.getSymbol()); + try { + LOG.debug("HEAD {} with change tracker {}", key, changeTracker); + if (changeTracker != null) { + changeTracker.maybeApplyConstraint(requestBuilder); + } + HeadObjectResponse headObjectResponse = + getS3Client().headObject(requestBuilder.build()); + if (fsHandler != null) { + long length = + fsHandler.getS3ObjectSize(key, headObjectResponse.contentLength(), this, + headObjectResponse); + // overwrite the content length + headObjectResponse = headObjectResponse.toBuilder().contentLength(length).build(); + } + if (changeTracker != null) { + changeTracker.processMetadata(headObjectResponse, operation); + } + return headObjectResponse; + } catch (AwsServiceException ase) { + if (!isObjectNotFound(ase)) { + // file not found is not considered a failure of the call, + // so only switch the duration tracker to update failure + // metrics on other exception outcomes. + duration.failed(); + } + throw ase; + } finally { + // update the tracker. + duration.close(); + } + }); + incrementReadOperations(); + return response; + } + + /** + * Retrieves a specific byte range of an S3 object as a stream. + * + * @param key The S3 object key to retrieve + * @param start The starting byte position (inclusive) of the range to retrieve + * @param end The ending byte position (inclusive) of the range to retrieve + * @return A ResponseInputStream containing the requested byte range of the S3 object + * @throws IOException If the object cannot be retrieved other I/O errors occur + * @see GetObjectResponse For additional metadata about the retrieved object + */ + @Override + @Retries.RetryRaw + public ResponseInputStream getRangedS3Object(String key, + long start, + long end) throws IOException { + final GetObjectRequest request = getRequestFactory().newGetObjectRequestBuilder(key) + .range(S3AUtils.formatRange(start, end)) + .build(); + DurationTracker duration = getDurationTrackerFactory() + .trackDuration(ACTION_HTTP_GET_REQUEST); + ResponseInputStream objectRange; + try { + objectRange = getStoreContext().getInvoker() + .retryUntranslated("GET Ranged Object " + key, true, + () -> getS3Client().getObject(request)); + + } catch (IOException ex) { + duration.failed(); + throw ex; + } finally { + duration.close(); + } + return objectRange; + } + /** * {@inheritDoc}. */ diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/prefetch/S3ARemoteObject.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/prefetch/S3ARemoteObject.java index ec6e3700226e08..b1f544e1ae70a5 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/prefetch/S3ARemoteObject.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/prefetch/S3ARemoteObject.java @@ -187,7 +187,7 @@ public ResponseInputStream openForRead(long offset, int size) streamStatistics.streamOpened(); final GetObjectRequest request = client .newGetRequestBuilder(s3Attributes.getKey()) - .range(S3AUtils.formatRange(offset, offset + size - 1)) + . range(S3AUtils.formatRange(offset, offset + size - 1)) .applyMutation(changeTracker::maybeApplyConstraint) .build(); diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/encryption.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/encryption.md index 674e987e278ede..9ad9ea9698dabe 100644 --- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/encryption.md +++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/encryption.md @@ -705,18 +705,30 @@ clients where S3-CSE has not been enabled. ### Features - Supports client side encryption with keys managed in AWS KMS (CSE-KMS) -- Supports client side encryption with custom keys by implementing custom [Keyring](https://docs.aws.amazon.com/encryption-sdk/latest/developer-guide/choose-keyring.html) (CSE-CUSTOM) -- Backward compatible with older encryption clients like `AmazonS3EncryptionClient.java`(V1) and `AmazonS3EncryptionClientV2.java`(V2) +- Supports client side encryption with custom keys by +implementing custom [Keyring](https://docs.aws.amazon.com/encryption-sdk/latest/developer-guide/choose-keyring.html) (CSE-CUSTOM) +- Backward compatible with older encryption clients +like `AmazonS3EncryptionClient.java`(V1) and `AmazonS3EncryptionClientV2.java`(V2) - encryption settings propagated into jobs through any issued delegation tokens. - encryption information stored as headers in the uploaded object. ### Compatibility Issues -- The V1 and V2 clients support reading unencrypted S3 objects, whereas the V3 client does not. In order to read S3 objects in a directory with a mix of encrypted and unencrypted objects. -- Unlike the V2 and V3 clients which always pads 16 bytes, V1 client pads extra bytes to the next multiple of 16. For example if unencrypted object size is 12bytes, V1 client pads extra 4bytes to make it multiple of 16. -- The V1 client supports storing encryption metadata in instruction file. +- The V1 client support reading unencrypted S3 objects, whereas the V3 +client does not. +- Unlike the V2 and V3 clients, which always append 16 bytes to a file, +the V1 client appends extra bytes to the next multiple of 16. +For example, if the unencrypted object size is 28 bytes, +the V1 client pads an extra 4 bytes to make it a multiple of 16. -Inorder to workaround the above compatibility issues set `fs.s3a.encryption.cse.v1.compatibility.enabled=true` +Inorder to workaround the above compatibility issues +set `fs.s3a.encryption.cse.v1.compatibility.enabled=true` + +Note: The V1 client supports storing encryption metadata in a separate file with +the suffix .instruction. However, these instruction files are not +skipped and will lead to exceptions or unknown issues. +Therefore, it is recommended not to use client-side encryption (CSE) +when instruction files are used to store encryption metadata. ### Limitations @@ -738,6 +750,7 @@ Inorder to workaround the above compatibility issues set `fs.s3a.encryption.cse. - If already created, [view the kms key ID by these steps.](https://docs.aws.amazon.com/kms/latest/developerguide/find-cmk-id-arn.html) - Set `fs.s3a.encryption.algorithm=CSE-KMS`. - Set `fs.s3a.encryption.key=`. +- Set `fs.s3a.encryption.cse.kms.region=` KMS_KEY_ID: @@ -766,11 +779,17 @@ S3-CSE to work. fs.s3a.encryption.key ${KMS_KEY_ID} + + +fs.s3a.encryption.cse.kms.region +${KMS_REGION} + ``` #### 2. CSE-CUSTOM - Set `fs.s3a.encryption.algorithm=CSE-CUSTOM`. -- Set `fs.s3a.encryption.cse.custom.cryptographic.material.manager.class.name=`. +- Set +`fs.s3a.encryption.cse.custom.cryptographic.material.manager.class.name=`. Example for custom keyring implementation ``` diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/troubleshooting_s3a.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/troubleshooting_s3a.md index 4856b0f576026e..e7796e1e97ec70 100644 --- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/troubleshooting_s3a.md +++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/troubleshooting_s3a.md @@ -1051,15 +1051,27 @@ file using configured SSE-C keyB into that structure. ## S3 Client Side Encryption -### Instruction file not found for S3 object +### java.lang.NoClassDefFoundError: software/amazon/encryption/s3/S3EncryptionClient -Reading an unencrypted file would fail when read through CSE enabled client. -``` -java.lang.SecurityException: Instruction file not found for S3 object with bucket name: ap-south-cse, key: unencryptedData.txt +With the move to the V2 AWS SDK, CSE is implemented via +[amazon-s3-encryption-client-java]( +https://github.com/aws/amazon-s3-encryption-client-java/tree/v3.1.1) +which is not packaged in AWS SDK V2 bundle jar and needs to be added seperately + + +Fix: add amazon-s3-encryption-client-java jar version 3.1.1 to the class path. + +### Instruction file not found for S3 object +Reading an unencrypted file would fail when read through CSE enabled client by default. ``` +software.amazon.encryption.s3.S3EncryptionClientException: Instruction file not found! +Please ensure the object you are attempting to decrypt has been encrypted +using the S3 Encryption Client. +``` CSE enabled client should read encrypted data only. +Fix: set `fs.s3a.encryption.cse.v1.compatibility.enabled=true` ### CSE-KMS method requires KMS key ID KMS key ID is required for CSE-KMS to encrypt data, not providing one leads @@ -1107,49 +1119,8 @@ Key 'arn:aws:kms:ap-south-1:152813717728:key/' does not exist(Service: AWSKMS; Status Code: 400; Error Code: NotFoundException; Request ID: 279db85d-864d-4a38-9acd-d892adb504c0; Proxy: null) ``` -While generating the KMS Key ID make sure to generate it in the same region - as your bucket. - -### Unable to perform range get request: Range get support has been disabled - -If Range get is not supported for a CSE algorithm or is disabled: -``` -java.lang.SecurityException: Unable to perform range get request: Range get support has been disabled. See https://docs.aws.amazon.com/general/latest/gr/aws_sdk_cryptography.html - -``` -Range gets must be enabled for CSE to work. - -### WARNING: Range gets do not provide authenticated encryption properties even when used with an authenticated mode (AES-GCM). - -The S3 Encryption Client is configured to support range get requests. This - warning would be shown everytime S3-CSE is used. -``` -2021-07-14 12:54:09,525 [main] WARN s3.AmazonS3EncryptionClientV2 -(AmazonS3EncryptionClientV2.java:warnOnRangeGetsEnabled(401)) - The S3 -Encryption Client is configured to support range get requests. Range gets do -not provide authenticated encryption properties even when used with an -authenticated mode (AES-GCM). See https://docs.aws.amazon.com/general/latest -/gr/aws_sdk_cryptography.html -``` -We can Ignore this warning since, range gets must be enabled for S3-CSE to -get data. - -### WARNING: If you don't have objects encrypted with these legacy modes, you should disable support for them to enhance security. - -The S3 Encryption Client is configured to read encrypted data with legacy -encryption modes through the CryptoMode setting, and we would see this -warning for all S3-CSE request. - -``` -2021-07-14 12:54:09,519 [main] WARN s3.AmazonS3EncryptionClientV2 -(AmazonS3EncryptionClientV2.java:warnOnLegacyCryptoMode(409)) - The S3 -Encryption Client is configured to read encrypted data with legacy -encryption modes through the CryptoMode setting. If you don't have objects -encrypted with these legacy modes, you should disable support for them to -enhance security. See https://docs.aws.amazon.com/general/latest/gr/aws_sdk_cryptography.html -``` -We can ignore this, since this CryptoMode setting(CryptoMode.AuthenticatedEncryption) -is required for range gets to work. +If S3 bucket region is different from the KMS key region, +set`fs.s3a.encryption.cse.kms.region=` ### `software.amazon.awssdk.services.kms.mode.InvalidKeyUsageException: You cannot generate a data key with an asymmetric CMK` diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/CustomKeyring.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/CustomKeyring.java index bbc0bd3d5d1eb0..564e9351feef80 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/CustomKeyring.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/CustomKeyring.java @@ -32,11 +32,11 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; -import static org.apache.hadoop.fs.s3a.Constants.AWS_REGION; import static org.apache.hadoop.fs.s3a.Constants.AWS_S3_DEFAULT_REGION; +import static org.apache.hadoop.fs.s3a.Constants.S3_ENCRYPTION_CSE_KMS_REGION; /** - * Custom Keyring implementation by warpping over KmsKeyring. + * Custom Keyring implementation. * This is used for testing {@link ITestS3AClientSideEncryptionCustom}. */ public class CustomKeyring implements Keyring { @@ -49,7 +49,7 @@ public CustomKeyring(Configuration conf) throws IOException { this.conf = conf; String bucket = S3ATestUtils.getFsName(conf); kmsClient = KmsClient.builder() - .region(Region.of(conf.get(AWS_REGION, AWS_S3_DEFAULT_REGION))) + .region(Region.of(conf.get(S3_ENCRYPTION_CSE_KMS_REGION, AWS_S3_DEFAULT_REGION))) .credentialsProvider(new TemporaryAWSCredentialsProvider( new Path(bucket).toUri(), conf)) .build(); diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AClientSideEncryption.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AClientSideEncryption.java index c8436502d8a902..1d9958866a368b 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AClientSideEncryption.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AClientSideEncryption.java @@ -53,7 +53,6 @@ import static org.apache.hadoop.fs.s3a.Constants.MULTIPART_MIN_SIZE; import static org.apache.hadoop.fs.s3a.Constants.S3_ENCRYPTION_ALGORITHM; import static org.apache.hadoop.fs.s3a.Constants.S3_ENCRYPTION_CSE_V1_COMPATIBILITY_ENABLED; -import static org.apache.hadoop.fs.s3a.Constants.S3_ENCRYPTION_CSE_INSTRUCTION_FILE_SUFFIX; import static org.apache.hadoop.fs.s3a.Constants.S3_ENCRYPTION_KEY; import static org.apache.hadoop.fs.s3a.Constants.SERVER_SIDE_ENCRYPTION_ALGORITHM; import static org.apache.hadoop.fs.s3a.Constants.SERVER_SIDE_ENCRYPTION_KEY; @@ -282,80 +281,34 @@ public void testUnencryptedObjectReadWithV1CompatibilityConfig() throws Exceptio maybeSkipTest(); // initialize base s3 client. Configuration conf = new Configuration(getConfiguration()); - S3AFileSystem nonCseFs = createTestFileSystem(conf); removeBaseAndBucketOverrides(getTestBucketName(conf), conf, S3_ENCRYPTION_ALGORITHM, S3_ENCRYPTION_KEY, SERVER_SIDE_ENCRYPTION_ALGORITHM, SERVER_SIDE_ENCRYPTION_KEY); - nonCseFs.initialize(getFileSystem().getUri(), conf); - Path file = path(getMethodName()); - // write unencrypted file - ContractTestUtils.writeDataset(nonCseFs, file, new byte[SMALL_FILE_SIZE], - SMALL_FILE_SIZE, SMALL_FILE_SIZE, true); + Path file = methodPath(); - Configuration cseConf = new Configuration(getConfiguration()); - cseConf.setBoolean(S3_ENCRYPTION_CSE_V1_COMPATIBILITY_ENABLED, true); - // create filesystem with cse enabled and v1 compatibility. - S3AFileSystem cseFs = createTestFileSystem(cseConf); - cseFs.initialize(getFileSystem().getUri(), cseConf); - - // read unencrypted file. It should not throw any exception. - try (FSDataInputStream in = cseFs.open(file)) { - in.read(new byte[SMALL_FILE_SIZE]); - } finally { - // close the filesystem - nonCseFs.close(); - cseFs.close(); - } - } - - /** - * Test to check if file name with suffix ".instruction" is ignored with V1 compatibility. - * @throws IOException - */ - @Test - public void testSkippingCSEInstructionFileWithV1Compatibility() throws IOException { - maybeSkipTest(); - // initialize base s3 client. - Configuration conf = new Configuration(getConfiguration()); - S3AFileSystem fs = createTestFileSystem(conf); - removeBaseAndBucketOverrides(getTestBucketName(conf), - conf, - S3_ENCRYPTION_ALGORITHM, - S3_ENCRYPTION_KEY, - SERVER_SIDE_ENCRYPTION_ALGORITHM, - SERVER_SIDE_ENCRYPTION_KEY); - fs.initialize(getFileSystem().getUri(), conf); + try (S3AFileSystem nonCseFs = createTestFileSystem(conf)) { + nonCseFs.initialize(getFileSystem().getUri(), conf); - // write file with suffix ".instruction" - Path filePath = path(getMethodName()); - Path file = new Path(filePath, - "file" + S3_ENCRYPTION_CSE_INSTRUCTION_FILE_SUFFIX); - ContractTestUtils.writeDataset(fs, file, new byte[SMALL_FILE_SIZE], - SMALL_FILE_SIZE, SMALL_FILE_SIZE, true); + // write unencrypted file + ContractTestUtils.writeDataset(nonCseFs, file, new byte[SMALL_FILE_SIZE], + SMALL_FILE_SIZE, SMALL_FILE_SIZE, true); + } - // create filesystem with cse enabled and v1 compatibility. Configuration cseConf = new Configuration(getConfiguration()); cseConf.setBoolean(S3_ENCRYPTION_CSE_V1_COMPATIBILITY_ENABLED, true); - S3AFileSystem cseFs = createTestFileSystem(cseConf); - cseFs.initialize(getFileSystem().getUri(), cseConf); - try { - // listing from fs without cse - Assertions.assertThat(fs.listStatus(filePath)).describedAs( - "Number of files aren't the same " + "as expected from FileStatus dir") - .hasSize(1); - - // listing fs without cse with v1 compatibility - Assertions.assertThat(cseFs.listStatus(filePath)).describedAs( - "Number of files aren't the same " + "as expected from FileStatus dir") - .hasSize(0); - } finally { - // close the filesystem - fs.close(); - cseFs.close(); + + // create filesystem with cse enabled and v1 compatibility. + try (S3AFileSystem cseFs = createTestFileSystem(cseConf)) { + cseFs.initialize(getFileSystem().getUri(), cseConf); + + // read unencrypted file. It should not throw any exception. + try (FSDataInputStream in = cseFs.open(file)) { + in.read(new byte[SMALL_FILE_SIZE]); + } } } @@ -369,36 +322,34 @@ public void testSizeOfEncryptedObjectFromHeaderWithV1Compatibility() throws Exce maybeSkipTest(); Configuration cseConf = new Configuration(getConfiguration()); cseConf.setBoolean(S3_ENCRYPTION_CSE_V1_COMPATIBILITY_ENABLED, true); - S3AFileSystem fs = createTestFileSystem(cseConf); - fs.initialize(getFileSystem().getUri(), cseConf); - - Path filePath = path(getMethodName()); - Path file = new Path(filePath, "file"); - String key = fs.pathToKey(file); - - - // write object with random content length header - Map metadata = new HashMap<>(); - metadata.put(AWSHeaders.UNENCRYPTED_CONTENT_LENGTH, "10"); - try (AuditSpan span = span()) { - RequestFactory factory = RequestFactoryImpl.builder() - .withBucket(fs.getBucket()) - .build(); - PutObjectRequest.Builder putObjectRequestBuilder = - factory.newPutObjectRequestBuilder(key, null, SMALL_FILE_SIZE, - false); - putObjectRequestBuilder.contentLength(Long.parseLong(String.valueOf(SMALL_FILE_SIZE))); - putObjectRequestBuilder.metadata(metadata); - fs.putObjectDirect(putObjectRequestBuilder.build(), - PutObjectOptions.deletingDirs(), - new S3ADataBlocks.BlockUploadData(new byte[SMALL_FILE_SIZE], null), - null); - - // fetch the random content length - long contentLength = fs.getFileStatus(file).getLen(); - assertEquals("content length does not match", 10, contentLength); - } finally { - fs.close(); + try (S3AFileSystem fs = createTestFileSystem(cseConf)) { + fs.initialize(getFileSystem().getUri(), cseConf); + + Path filePath = methodPath(); + Path file = new Path(filePath, "file"); + String key = fs.pathToKey(file); + + // write object with random content length header + Map metadata = new HashMap<>(); + metadata.put(AWSHeaders.UNENCRYPTED_CONTENT_LENGTH, "10"); + try (AuditSpan span = span()) { + RequestFactory factory = RequestFactoryImpl.builder() + .withBucket(fs.getBucket()) + .build(); + PutObjectRequest.Builder putObjectRequestBuilder = + factory.newPutObjectRequestBuilder(key, + null, SMALL_FILE_SIZE, false); + putObjectRequestBuilder.contentLength(Long.parseLong(String.valueOf(SMALL_FILE_SIZE))); + putObjectRequestBuilder.metadata(metadata); + fs.putObjectDirect(putObjectRequestBuilder.build(), + PutObjectOptions.deletingDirs(), + new S3ADataBlocks.BlockUploadData(new byte[SMALL_FILE_SIZE], null), + null); + + // fetch the random content length + long contentLength = fs.getFileStatus(file).getLen(); + assertEquals("content length does not match", 10, contentLength); + } } } @@ -410,40 +361,32 @@ public void testSizeOfEncryptedObjectFromHeaderWithV1Compatibility() throws Exce @Test public void testSizeOfUnencryptedObjectWithV1Compatibility() throws Exception { maybeSkipTest(); - // initialize base s3 client. Configuration conf = new Configuration(getConfiguration()); - S3AFileSystem fs = createTestFileSystem(conf); - removeBaseAndBucketOverrides(getTestBucketName(conf), - conf, - S3_ENCRYPTION_ALGORITHM, - S3_ENCRYPTION_KEY, - SERVER_SIDE_ENCRYPTION_ALGORITHM, - SERVER_SIDE_ENCRYPTION_KEY); - fs.initialize(getFileSystem().getUri(), conf); + conf.setBoolean(S3_ENCRYPTION_CSE_V1_COMPATIBILITY_ENABLED, false); + Path file = methodPath(); + try (S3AFileSystem fs = createTestFileSystem(conf)) { + fs.initialize(getFileSystem().getUri(), conf); - Path file = path(getMethodName()); - // Unencrypted data written to a path. - ContractTestUtils.writeDataset(fs, file, new byte[SMALL_FILE_SIZE], - SMALL_FILE_SIZE, SMALL_FILE_SIZE, true); + // Unencrypted data written to a path. + ContractTestUtils.writeDataset(fs, file, new byte[SMALL_FILE_SIZE], SMALL_FILE_SIZE, + SMALL_FILE_SIZE, true); + + // check the file size + FileStatus status1 = fs.getFileStatus(file); + assertEquals("Mismatch in content length bytes", SMALL_FILE_SIZE, status1.getLen()); + } // initialize encrypted s3 client with support for reading unencrypted objects Configuration cseConf = new Configuration(getConfiguration()); cseConf.setBoolean(S3_ENCRYPTION_CSE_V1_COMPATIBILITY_ENABLED, true); - S3AFileSystem cseFs = createTestFileSystem(cseConf); - cseFs.initialize(getFileSystem().getUri(), cseConf); - // check the file size - try { - FileStatus status1 = fs.getFileStatus(file); - assertEquals("Mismatch in content length bytes", SMALL_FILE_SIZE, - status1.getLen()); + try (S3AFileSystem cseFs = createTestFileSystem(cseConf)) { + cseFs.initialize(getFileSystem().getUri(), cseConf); + + // check the file size FileStatus status2 = cseFs.getFileStatus(file); assertEquals("Mismatch in content length bytes", SMALL_FILE_SIZE, status2.getLen()); - } finally { - // close the filesystem - fs.close(); - cseFs.close(); } } @@ -457,22 +400,17 @@ public void testSizeOfEncryptedObjectWithV1Compatibility() throws Exception { maybeSkipTest(); Configuration cseConf = new Configuration(getConfiguration()); cseConf.setBoolean(S3_ENCRYPTION_CSE_V1_COMPATIBILITY_ENABLED, true); - S3AFileSystem fs = createTestFileSystem(cseConf); - fs.initialize(getFileSystem().getUri(), cseConf); + try (S3AFileSystem fs = createTestFileSystem(cseConf)) { + fs.initialize(getFileSystem().getUri(), cseConf); - // write encrypted file - Path file = path(getMethodName()); - ContractTestUtils.writeDataset(fs, file, new byte[SMALL_FILE_SIZE], - SMALL_FILE_SIZE, SMALL_FILE_SIZE, true); + // write encrypted file + Path file = methodPath(); + ContractTestUtils.writeDataset(fs, file, new byte[SMALL_FILE_SIZE], SMALL_FILE_SIZE, + SMALL_FILE_SIZE, true); - // check the file size - try { FileStatus status2 = fs.getFileStatus(file); - assertEquals("Mismatch in content length bytes", SMALL_FILE_SIZE, - status2.getLen()); - } finally { - // close the filesystem - fs.close(); + assertEquals("Mismatch in content length bytes", + SMALL_FILE_SIZE, status2.getLen()); } } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AClientSideEncryptionCustom.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AClientSideEncryptionCustom.java index c94b804a1ec566..bc7a7b5a5be882 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AClientSideEncryptionCustom.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AClientSideEncryptionCustom.java @@ -31,7 +31,7 @@ import static org.apache.hadoop.fs.s3a.S3ATestUtils.skipIfEncryptionTestsDisabled; /** - * Tests to verify S3 client side encryption (CSE-CUSTOM). + * Tests to verify Custom S3 client side encryption CSE-CUSTOM. */ public class ITestS3AClientSideEncryptionCustom extends ITestS3AClientSideEncryption { diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/ITestConnectionTimeouts.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/ITestConnectionTimeouts.java index 362c8a8ac1735b..cb049cbd2f1ec0 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/ITestConnectionTimeouts.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/ITestConnectionTimeouts.java @@ -194,6 +194,7 @@ public void testGeneratePoolTimeouts() throws Throwable { */ @Test public void testObjectUploadTimeouts() throws Throwable { + skipIfClientSideEncryption(); AWSClientConfig.setMinimumOperationDuration(Duration.ZERO); final Path dir = methodPath(); Path file = new Path(dir, "file"); diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/test/MinimalListingOperationCallbacks.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/test/MinimalListingOperationCallbacks.java index e4d76faff7eb63..504f701d00084d 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/test/MinimalListingOperationCallbacks.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/test/MinimalListingOperationCallbacks.java @@ -21,6 +21,8 @@ import java.io.IOException; import java.util.concurrent.CompletableFuture; +import software.amazon.awssdk.services.s3.model.S3Object; + import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.s3a.S3AFileStatus; import org.apache.hadoop.fs.s3a.S3ALocatedFileStatus; @@ -30,8 +32,6 @@ import org.apache.hadoop.fs.statistics.DurationTrackerFactory; import org.apache.hadoop.fs.store.audit.AuditSpan; -import software.amazon.awssdk.services.s3.model.S3Object; - /** * Stub implementation of {@link ListingOperationCallbacks}. */ From b1969658f4383b62cb0d4ec303580c0620a24fb6 Mon Sep 17 00:00:00 2001 From: Syed Shameerur Rahman Date: Fri, 1 Nov 2024 08:02:47 +0530 Subject: [PATCH 3/6] fix checkstyle --- .../fs/s3a/prefetch/S3ARemoteObject.java | 2 +- .../markdown/tools/hadoop-aws/encryption.md | 33 +++++++++---------- .../tools/hadoop-aws/troubleshooting_s3a.md | 12 +++---- .../fs/s3a/impl/TestRequestFactory.java | 2 +- 4 files changed, 22 insertions(+), 27 deletions(-) diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/prefetch/S3ARemoteObject.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/prefetch/S3ARemoteObject.java index b1f544e1ae70a5..ec6e3700226e08 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/prefetch/S3ARemoteObject.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/prefetch/S3ARemoteObject.java @@ -187,7 +187,7 @@ public ResponseInputStream openForRead(long offset, int size) streamStatistics.streamOpened(); final GetObjectRequest request = client .newGetRequestBuilder(s3Attributes.getKey()) - . range(S3AUtils.formatRange(offset, offset + size - 1)) + .range(S3AUtils.formatRange(offset, offset + size - 1)) .applyMutation(changeTracker::maybeApplyConstraint) .build(); diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/encryption.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/encryption.md index 9ad9ea9698dabe..1d598f37599d58 100644 --- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/encryption.md +++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/encryption.md @@ -703,31 +703,28 @@ shorter than the length of files listed with other clients -including S3A clients where S3-CSE has not been enabled. ### Features - - Supports client side encryption with keys managed in AWS KMS (CSE-KMS) -- Supports client side encryption with custom keys by +- Supports client side encryption with custom keys by implementing custom [Keyring](https://docs.aws.amazon.com/encryption-sdk/latest/developer-guide/choose-keyring.html) (CSE-CUSTOM) -- Backward compatible with older encryption clients +- Backward compatible with older encryption clients like `AmazonS3EncryptionClient.java`(V1) and `AmazonS3EncryptionClientV2.java`(V2) - encryption settings propagated into jobs through any issued delegation tokens. - encryption information stored as headers in the uploaded object. ### Compatibility Issues - -- The V1 client support reading unencrypted S3 objects, whereas the V3 -client does not. -- Unlike the V2 and V3 clients, which always append 16 bytes to a file, -the V1 client appends extra bytes to the next multiple of 16. -For example, if the unencrypted object size is 28 bytes, +- The V1 client support reading unencrypted S3 objects, whereas the V3 client does not. +- Unlike the V2 and V3 clients, which always append 16 bytes to a file, +the V1 client appends extra bytes to the next multiple of 16. +For example, if the unencrypted object size is 28 bytes, the V1 client pads an extra 4 bytes to make it a multiple of 16. -Inorder to workaround the above compatibility issues +Note: Inorder to workaround the above compatibility issues set `fs.s3a.encryption.cse.v1.compatibility.enabled=true` -Note: The V1 client supports storing encryption metadata in a separate file with -the suffix .instruction. However, these instruction files are not -skipped and will lead to exceptions or unknown issues. -Therefore, it is recommended not to use client-side encryption (CSE) +Note: The V1 client supports storing encryption metadata in a separate file with +the suffix "fileName".instruction. However, these instruction files are not +skipped and will lead to exceptions or unknown issues. +Therefore, it is recommended not to use S3A client-side encryption (CSE) when instruction files are used to store encryption metadata. ### Limitations @@ -750,7 +747,7 @@ when instruction files are used to store encryption metadata. - If already created, [view the kms key ID by these steps.](https://docs.aws.amazon.com/kms/latest/developerguide/find-cmk-id-arn.html) - Set `fs.s3a.encryption.algorithm=CSE-KMS`. - Set `fs.s3a.encryption.key=`. -- Set `fs.s3a.encryption.cse.kms.region=` +- Set `fs.s3a.encryption.cse.kms.region=`. KMS_KEY_ID: @@ -781,14 +778,14 @@ S3-CSE to work. -fs.s3a.encryption.cse.kms.region -${KMS_REGION} + fs.s3a.encryption.cse.kms.region + ${KMS_REGION} ``` #### 2. CSE-CUSTOM - Set `fs.s3a.encryption.algorithm=CSE-CUSTOM`. -- Set +- Set `fs.s3a.encryption.cse.custom.cryptographic.material.manager.class.name=`. Example for custom keyring implementation diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/troubleshooting_s3a.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/troubleshooting_s3a.md index e7796e1e97ec70..e33c7b181e4ca8 100644 --- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/troubleshooting_s3a.md +++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/troubleshooting_s3a.md @@ -1053,11 +1053,9 @@ file using configured SSE-C keyB into that structure. ### java.lang.NoClassDefFoundError: software/amazon/encryption/s3/S3EncryptionClient -With the move to the V2 AWS SDK, CSE is implemented via -[amazon-s3-encryption-client-java]( -https://github.com/aws/amazon-s3-encryption-client-java/tree/v3.1.1) -which is not packaged in AWS SDK V2 bundle jar and needs to be added seperately - +With the move to the V2 AWS SDK, CSE is implemented via +[amazon-s3-encryption-client-java](https://github.com/aws/amazon-s3-encryption-client-java/tree/v3.1.1) +which is not packaged in AWS SDK V2 bundle jar and needs to be added separately. Fix: add amazon-s3-encryption-client-java jar version 3.1.1 to the class path. @@ -1065,8 +1063,8 @@ Fix: add amazon-s3-encryption-client-java jar version 3.1.1 to the class path. Reading an unencrypted file would fail when read through CSE enabled client by default. ``` -software.amazon.encryption.s3.S3EncryptionClientException: Instruction file not found! -Please ensure the object you are attempting to decrypt has been encrypted +software.amazon.encryption.s3.S3EncryptionClientException: Instruction file not found! +Please ensure the object you are attempting to decrypt has been encrypted using the S3 Encryption Client. ``` CSE enabled client should read encrypted data only. diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestRequestFactory.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestRequestFactory.java index 321f47f80ec270..c779062f518f78 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestRequestFactory.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestRequestFactory.java @@ -270,7 +270,7 @@ public void testUploadTimeouts() throws Throwable { // multipart part final UploadPartRequest upload = factory.newUploadPartRequestBuilder(path, - "1", 3, false,128_000_000) + "1", 3, false, 128_000_000) .build(); assertApiTimeouts(partDuration, upload); From 59a7e05efb8e303084eef8022a1fea9dcfb941c2 Mon Sep 17 00:00:00 2001 From: Syed Shameerur Rahman Date: Fri, 1 Nov 2024 15:48:48 +0530 Subject: [PATCH 4/6] checkstyle fix --- .../src/site/markdown/tools/hadoop-aws/troubleshooting_s3a.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/troubleshooting_s3a.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/troubleshooting_s3a.md index e33c7b181e4ca8..88f5fcf3d02131 100644 --- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/troubleshooting_s3a.md +++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/troubleshooting_s3a.md @@ -1066,7 +1066,7 @@ Reading an unencrypted file would fail when read through CSE enabled client by d software.amazon.encryption.s3.S3EncryptionClientException: Instruction file not found! Please ensure the object you are attempting to decrypt has been encrypted using the S3 Encryption Client. -``` +``` CSE enabled client should read encrypted data only. Fix: set `fs.s3a.encryption.cse.v1.compatibility.enabled=true` From 2f2790f5d2f3ae18ad32f991a6f549eefd1dda79 Mon Sep 17 00:00:00 2001 From: Syed Shameerur Rahman Date: Wed, 6 Nov 2024 18:51:17 +0530 Subject: [PATCH 5/6] Fix PR review comments --- .../hadoop/fs/s3a/DefaultS3ClientFactory.java | 2 +- .../apache/hadoop/fs/s3a/S3AFileSystem.java | 22 +++--- .../org/apache/hadoop/fs/s3a/S3AStore.java | 4 +- .../org/apache/hadoop/fs/s3a/S3AUtils.java | 3 + ....java => BaseS3AFileSystemOperations.java} | 8 +-- ...r.java => CSES3AFileSystemOperations.java} | 8 +-- ...EV1CompatibleS3AFileSystemOperations.java} | 18 ++--- .../s3a/impl/EncryptionS3ClientFactory.java | 69 +++++++++++++------ .../hadoop/fs/s3a/impl/ErrorTranslation.java | 55 +++++++++++++++ ...dler.java => S3AFileSystemOperations.java} | 2 +- .../hadoop/fs/s3a/impl/S3AStoreImpl.java | 4 +- .../fs/s3a/ITestS3AClientSideEncryption.java | 62 +++++++++-------- .../ITestS3AClientSideEncryptionCustom.java | 11 ++- .../hadoop/fs/s3a/ITestS3AConfiguration.java | 11 --- .../hadoop/fs/s3a/ITestS3ADelayedFNF.java | 11 +-- .../fs/s3a/ITestS3AFailureHandling.java | 14 +--- .../apache/hadoop/fs/s3a/S3ATestUtils.java | 30 ++++++++ .../fs/s3a/impl/ITestConnectionTimeouts.java | 11 +-- .../fs/s3a/impl/TestErrorTranslation.java | 33 ++++++++- 19 files changed, 247 insertions(+), 131 deletions(-) rename hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/{BaseS3AFileSystemHandler.java => BaseS3AFileSystemOperations.java} (94%) rename hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/{CSES3AFileSystemHandler.java => CSES3AFileSystemOperations.java} (94%) rename hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/{CSEV1CompatibleS3AFileSystemHandler.java => CSEV1CompatibleS3AFileSystemOperations.java} (94%) rename hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/{S3AFileSystemHandler.java => S3AFileSystemOperations.java} (98%) diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/DefaultS3ClientFactory.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/DefaultS3ClientFactory.java index 1e5a5648ecb42a..556bd3510752ea 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/DefaultS3ClientFactory.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/DefaultS3ClientFactory.java @@ -165,7 +165,7 @@ public S3AsyncClient createS3AsyncClient( configureClientBuilder(S3AsyncClient.builder(), parameters, conf, bucket) .httpClientBuilder(httpClientBuilder); - // TODO: Enable multi part upload with cse once it is available. + // multipart upload pending with HADOOP-19326. if (!parameters.isClientSideEncryptionEnabled()) { s3AsyncClientBuilder.multipartConfiguration(multipartConfiguration) .multipartEnabled(parameters.isMultipartCopy()); diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java index 0c5b5343df5ef1..98469539111716 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java @@ -115,9 +115,10 @@ import org.apache.hadoop.fs.s3a.auth.delegation.DelegationTokenProvider; import org.apache.hadoop.fs.s3a.commit.magic.InMemoryMagicCommitTracker; import org.apache.hadoop.fs.s3a.impl.AWSCannedACL; -import org.apache.hadoop.fs.s3a.impl.BaseS3AFileSystemHandler; +import org.apache.hadoop.fs.s3a.impl.BaseS3AFileSystemOperations; import org.apache.hadoop.fs.s3a.impl.BulkDeleteOperation; import org.apache.hadoop.fs.s3a.impl.BulkDeleteOperationCallbacksImpl; +import org.apache.hadoop.fs.s3a.impl.CSES3AFileSystemOperations; import org.apache.hadoop.fs.s3a.impl.ChangeDetectionPolicy; import org.apache.hadoop.fs.s3a.impl.ClientManager; import org.apache.hadoop.fs.s3a.impl.ClientManagerImpl; @@ -125,9 +126,8 @@ import org.apache.hadoop.fs.s3a.impl.ContextAccessors; import org.apache.hadoop.fs.s3a.impl.CopyFromLocalOperation; import org.apache.hadoop.fs.s3a.impl.CreateFileBuilder; -import org.apache.hadoop.fs.s3a.impl.S3AFileSystemHandler; -import org.apache.hadoop.fs.s3a.impl.CSES3AFileSystemHandler; -import org.apache.hadoop.fs.s3a.impl.CSEV1CompatibleS3AFileSystemHandler; +import org.apache.hadoop.fs.s3a.impl.S3AFileSystemOperations; +import org.apache.hadoop.fs.s3a.impl.CSEV1CompatibleS3AFileSystemOperations; import org.apache.hadoop.fs.s3a.impl.CSEMaterials; import org.apache.hadoop.fs.s3a.impl.DeleteOperation; import org.apache.hadoop.fs.s3a.impl.DirectoryPolicy; @@ -468,7 +468,7 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities, /** * Handler for certain filesystem operations. */ - private S3AFileSystemHandler fsHandler; + private S3AFileSystemOperations fsHandler; /** @@ -830,21 +830,21 @@ public void initialize(URI name, Configuration originalConf) } /** - * Creates and returns an instance of the appropriate S3AFileSystemHandler. + * Creates and returns an instance of the appropriate S3AFileSystemOperations. * Creation is baaed on the client-side encryption (CSE) settings. * - * @return An instance of the appropriate S3AFileSystemHandler implementation. + * @return An instance of the appropriate S3AFileSystemOperations implementation. */ - private S3AFileSystemHandler createFileSystemHandler() { + private S3AFileSystemOperations createFileSystemHandler() { if (isCSEEnabled) { if (getConf().getBoolean(S3_ENCRYPTION_CSE_V1_COMPATIBILITY_ENABLED, S3_ENCRYPTION_CSE_V1_COMPATIBILITY_ENABLED_DEFAULT)) { - return new CSEV1CompatibleS3AFileSystemHandler(); + return new CSEV1CompatibleS3AFileSystemOperations(); } else { - return new CSES3AFileSystemHandler(); + return new CSES3AFileSystemOperations(); } } else { - return new BaseS3AFileSystemHandler(); + return new BaseS3AFileSystemOperations(); } } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AStore.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AStore.java index b10769dfe73f39..ab8785e01dafda 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AStore.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AStore.java @@ -49,7 +49,7 @@ import org.apache.hadoop.fs.s3a.impl.ChangeTracker; import org.apache.hadoop.fs.s3a.impl.ClientManager; import org.apache.hadoop.fs.s3a.impl.MultiObjectDeleteException; -import org.apache.hadoop.fs.s3a.impl.S3AFileSystemHandler; +import org.apache.hadoop.fs.s3a.impl.S3AFileSystemOperations; import org.apache.hadoop.fs.s3a.impl.StoreContext; import org.apache.hadoop.fs.s3a.statistics.S3AStatisticsContext; import org.apache.hadoop.fs.statistics.DurationTrackerFactory; @@ -213,7 +213,7 @@ Map.Entry> deleteObject( HeadObjectResponse headObject(String key, ChangeTracker changeTracker, Invoker changeInvoker, - S3AFileSystemHandler fsHandler, + S3AFileSystemOperations fsHandler, String operation) throws IOException; /** diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java index 741607c7b8ad37..fbc8d39d73a827 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java @@ -80,6 +80,7 @@ import static org.apache.hadoop.fs.s3a.Constants.*; import static org.apache.hadoop.fs.s3a.audit.AuditIntegration.maybeTranslateAuditException; import static org.apache.hadoop.fs.s3a.impl.ErrorTranslation.isUnknownBucket; +import static org.apache.hadoop.fs.s3a.impl.ErrorTranslation.maybeExtractSdkExceptionFromEncryptionClientException; import static org.apache.hadoop.fs.s3a.impl.InstantiationIOException.instantiationException; import static org.apache.hadoop.fs.s3a.impl.InstantiationIOException.isAbstract; import static org.apache.hadoop.fs.s3a.impl.InstantiationIOException.isNotInstanceOf; @@ -184,6 +185,8 @@ public static IOException translateException(@Nullable String operation, path = "/"; } + exception = maybeExtractSdkExceptionFromEncryptionClientException(exception); + if (!(exception instanceof AwsServiceException)) { // exceptions raised client-side: connectivity, auth, network problems... Exception innerCause = containsInterruptedException(exception); diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/BaseS3AFileSystemHandler.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/BaseS3AFileSystemOperations.java similarity index 94% rename from hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/BaseS3AFileSystemHandler.java rename to hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/BaseS3AFileSystemOperations.java index 2fc7447693db39..b06ff970d5a6e4 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/BaseS3AFileSystemHandler.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/BaseS3AFileSystemOperations.java @@ -38,15 +38,15 @@ import static org.apache.hadoop.fs.s3a.Statistic.CLIENT_SIDE_ENCRYPTION_ENABLED; /** - * An implementation of the {@link S3AFileSystemHandler} interface. + * An implementation of the {@link S3AFileSystemOperations} interface. * This handles certain filesystem operations when s3 client side encryption is disabled. */ -public class BaseS3AFileSystemHandler implements S3AFileSystemHandler { +public class BaseS3AFileSystemOperations implements S3AFileSystemOperations { /** - * Constructs a new instance of {@code BaseS3AFileSystemHandler}. + * Constructs a new instance of {@code BaseS3AFileSystemOperations}. */ - public BaseS3AFileSystemHandler() { + public BaseS3AFileSystemOperations() { } /** diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/CSES3AFileSystemHandler.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/CSES3AFileSystemOperations.java similarity index 94% rename from hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/CSES3AFileSystemHandler.java rename to hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/CSES3AFileSystemOperations.java index 8529d7b180c682..4cfab1b415972c 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/CSES3AFileSystemHandler.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/CSES3AFileSystemOperations.java @@ -37,15 +37,15 @@ import static org.apache.hadoop.fs.s3a.impl.InternalConstants.CSE_PADDING_LENGTH; /** - * An implementation of the {@link S3AFileSystemHandler} interface. + * An implementation of the {@link S3AFileSystemOperations} interface. * This handles certain filesystem operations when s3 client side encryption is enabled. */ -public class CSES3AFileSystemHandler implements S3AFileSystemHandler{ +public class CSES3AFileSystemOperations implements S3AFileSystemOperations { /** - * Constructs a new instance of {@code CSES3AFileSystemHandler}. + * Constructs a new instance of {@code CSES3AFileSystemOperations}. */ - public CSES3AFileSystemHandler() { + public CSES3AFileSystemOperations() { } /** diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/CSEV1CompatibleS3AFileSystemHandler.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/CSEV1CompatibleS3AFileSystemOperations.java similarity index 94% rename from hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/CSEV1CompatibleS3AFileSystemHandler.java rename to hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/CSEV1CompatibleS3AFileSystemOperations.java index 493357d4e8ccdb..3a540c2e5f3436 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/CSEV1CompatibleS3AFileSystemHandler.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/CSEV1CompatibleS3AFileSystemOperations.java @@ -20,33 +20,33 @@ import java.io.IOException; +import software.amazon.awssdk.core.ResponseInputStream; +import software.amazon.awssdk.services.s3.model.GetObjectRequest; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; +import software.amazon.awssdk.services.s3.model.HeadObjectResponse; + import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.s3a.S3AStore; import org.apache.hadoop.fs.s3a.S3ClientFactory; import org.apache.hadoop.fs.s3a.api.RequestFactory; import org.apache.hadoop.util.ReflectionUtils; -import software.amazon.awssdk.core.ResponseInputStream; -import software.amazon.awssdk.services.s3.model.GetObjectRequest; -import software.amazon.awssdk.services.s3.model.GetObjectResponse; -import software.amazon.awssdk.services.s3.model.HeadObjectResponse; - import static org.apache.hadoop.fs.s3a.Constants.DEFAULT_S3_CLIENT_FACTORY_IMPL; import static org.apache.hadoop.fs.s3a.Constants.S3_CLIENT_FACTORY_IMPL; import static org.apache.hadoop.fs.s3a.impl.CSEUtils.isObjectEncrypted; /** - * An extension of the {@link CSES3AFileSystemHandler} class. + * An extension of the {@link CSES3AFileSystemOperations} class. * This handles certain file system operations when client-side encryption is enabled with v1 client * compatibility. * {@link org.apache.hadoop.fs.s3a.Constants#S3_ENCRYPTION_CSE_V1_COMPATIBILITY_ENABLED}. */ -public class CSEV1CompatibleS3AFileSystemHandler extends CSES3AFileSystemHandler { +public class CSEV1CompatibleS3AFileSystemOperations extends CSES3AFileSystemOperations { /** - * Constructs a new instance of {@code CSEV1CompatibleS3AFileSystemHandler}. + * Constructs a new instance of {@code CSEV1CompatibleS3AFileSystemOperations}. */ - public CSEV1CompatibleS3AFileSystemHandler() { + public CSEV1CompatibleS3AFileSystemOperations() { } /** diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/EncryptionS3ClientFactory.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/EncryptionS3ClientFactory.java index 57f1173dc63749..122c4c2852333f 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/EncryptionS3ClientFactory.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/EncryptionS3ClientFactory.java @@ -21,12 +21,6 @@ import java.io.IOException; import java.net.URI; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.s3a.DefaultS3ClientFactory; -import org.apache.hadoop.util.Preconditions; -import org.apache.hadoop.util.ReflectionUtils; -import org.apache.hadoop.util.functional.LazyAtomicReference; - import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.services.kms.KmsClient; import software.amazon.awssdk.services.kms.KmsClientBuilder; @@ -39,6 +33,12 @@ import software.amazon.encryption.s3.materials.Keyring; import software.amazon.encryption.s3.materials.KmsKeyring; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.s3a.DefaultS3ClientFactory; +import org.apache.hadoop.util.Preconditions; +import org.apache.hadoop.util.ReflectionUtils; +import org.apache.hadoop.util.functional.LazyAtomicReference; + import static org.apache.hadoop.fs.s3a.impl.InstantiationIOException.unavailable; /** @@ -141,8 +141,10 @@ public S3AsyncClient createS3AsyncClient(URI uri, S3ClientCreationParameters par * * @param parameters The S3ClientCreationParameters containing the necessary configuration. * @return An instance of S3EncryptionClient. + * @throws IOException If an error occurs during the creation of the S3EncryptionClient. */ - private S3Client createS3EncryptionClient(S3ClientCreationParameters parameters) { + private S3Client createS3EncryptionClient(S3ClientCreationParameters parameters) + throws IOException { CSEMaterials cseMaterials = parameters.getClientSideEncryptionMaterials(); Preconditions.checkArgument(s3AsyncClient != null, "S3 async client not initialized"); @@ -170,8 +172,13 @@ private S3Client createS3EncryptionClient(S3ClientCreationParameters parameters) s3EncryptionClientBuilder.cryptoMaterialsManager(kmsCryptoMaterialsManager); break; case CUSTOM: - Keyring keyring = getKeyringProvider(cseMaterials.getCustomKeyringClassName(), - cseMaterials.getConf()); + Keyring keyring; + try { + keyring = + getKeyringProvider(cseMaterials.getCustomKeyringClassName(), cseMaterials.getConf()); + } catch (RuntimeException e) { + throw new IOException("Failed to instantiate a custom keyring provider", e); + } CryptographicMaterialsManager customCryptoMaterialsManager = DefaultCryptoMaterialsManager.builder() .keyring(keyring) @@ -220,8 +227,10 @@ private Keyring createKmsKeyring(S3ClientCreationParameters parameters, * * @param parameters The S3ClientCreationParameters containing the necessary configuration. * @return An instance of S3AsyncEncryptionClient. + * @throws IOException If an error occurs during the creation of the S3AsyncEncryptionClient. */ - private S3AsyncClient createS3AsyncEncryptionClient(S3ClientCreationParameters parameters) { + private S3AsyncClient createS3AsyncEncryptionClient(S3ClientCreationParameters parameters) + throws IOException { Preconditions.checkArgument(s3AsyncClient != null, "S3 async client not initialized"); Preconditions.checkArgument(parameters != null, @@ -246,8 +255,13 @@ private S3AsyncClient createS3AsyncEncryptionClient(S3ClientCreationParameters p s3EncryptionAsyncClientBuilder.cryptoMaterialsManager(kmsCryptoMaterialsManager); break; case CUSTOM: - Keyring keyring = getKeyringProvider(cseMaterials.getCustomKeyringClassName(), - cseMaterials.getConf()); + Keyring keyring; + try { + keyring = + getKeyringProvider(cseMaterials.getCustomKeyringClassName(), cseMaterials.getConf()); + } catch (RuntimeException e) { + throw new IOException("Failed to instantiate a custom keyring provider", e); + } CryptographicMaterialsManager customCryptoMaterialsManager = DefaultCryptoMaterialsManager.builder() .keyring(keyring) @@ -260,21 +274,32 @@ private S3AsyncClient createS3AsyncEncryptionClient(S3ClientCreationParameters p return s3EncryptionAsyncClientBuilder.build(); } - /** - * Retrieves an instance of the Keyring provider based on the provided class name. + * Creates and returns a Keyring provider instance based on the given class name. + * + *

This method attempts to instantiate a Keyring provider using reflection. It first tries + * to create an instance using the standard ReflectionUtils.newInstance method. If that fails, + * it falls back to an alternative instantiation method, which is primarily used for testing + * purposes (specifically for CustomKeyring.java). * - * @param className The fully qualified class name of the Keyring provider implementation. - * @param conf The Configuration object containing the necessary configuration properties. - * @return An instance of the Keyring provider. + * @param className The fully qualified class name of the Keyring provider to instantiate. + * @param conf The Configuration object to be passed to the Keyring provider constructor. + * @return An instance of the specified Keyring provider. + * @throws RuntimeException If unable to create the Keyring provider instance. */ - private Keyring getKeyringProvider(String className, Configuration conf) { + private Keyring getKeyringProvider(String className, Configuration conf) { + Class keyringProviderClass = getCustomKeyringProviderClass(className); try { - return ReflectionUtils.newInstance(getCustomKeyringProviderClass(className), conf); + return ReflectionUtils.newInstance(keyringProviderClass, conf); } catch (Exception e) { - // this is for testing purpose to support CustomKeyring.java - return ReflectionUtils.newInstance(getCustomKeyringProviderClass(className), conf, - new Class[] {Configuration.class}, conf); + LOG.warn("Failed to create Keyring provider", e); + // This is for testing purposes to support CustomKeyring.java + try { + return ReflectionUtils.newInstance(keyringProviderClass, conf, + new Class[] {Configuration.class}, conf); + } catch (Exception ex) { + throw new RuntimeException("Failed to create Keyring provider", ex); + } } } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/ErrorTranslation.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/ErrorTranslation.java index 7934a5c7d4d5c1..eeb13d1025a2b1 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/ErrorTranslation.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/ErrorTranslation.java @@ -22,6 +22,7 @@ import java.lang.reflect.Constructor; import software.amazon.awssdk.awscore.exception.AwsServiceException; +import software.amazon.awssdk.core.exception.SdkException; import org.apache.hadoop.classification.VisibleForTesting; import org.apache.hadoop.fs.s3a.HttpChannelEOFException; @@ -63,6 +64,12 @@ public final class ErrorTranslation { private static final String SHADED_NO_HTTP_RESPONSE_EXCEPTION = "software.amazon.awssdk.thirdparty.org.apache.http.NoHttpResponseException"; + /** + * S3 encryption client exception class name: {@value}. + */ + private static final String S3_ENCRYPTION_CLIENT_EXCEPTION = + "software.amazon.encryption.s3.S3EncryptionClientException"; + /** * Private constructor for utility class. */ @@ -105,6 +112,54 @@ private static Throwable getInnermostThrowable(Throwable thrown, Throwable outer return getInnermostThrowable(thrown.getCause(), thrown); } + /** + * Attempts to extract the underlying SdkException from an S3 encryption client exception. + * + *

This method is designed to handle exceptions that may be wrapped within + * S3EncryptionClientExceptions. It performs the following steps: + *

    + *
  1. Checks if the input exception is null.
    2. + *
  2. Verifies if the exception contains the S3EncryptionClientException signature.
    3. + *
  3. Examines the cause chain to find the most relevant SdkException.
    4. + *
+ * + *

The method aims to unwrap nested exceptions to provide more meaningful + * error information, particularly in the context of S3 encryption operations. + * + * @param exception The SdkException to analyze. This may be a wrapper exception + * containing a more specific underlying cause. + * @return The extracted SdkException if found within the exception chain, + * or the original exception if no relevant nested exception is found. + * Returns null if the input exception is null. + * + * @see SdkException + * @see AwsServiceException + */ + public static SdkException maybeExtractSdkExceptionFromEncryptionClientException( + SdkException exception) { + if (exception == null) { + return null; + } + + // check if the exception contains S3EncryptionClientException + if (!exception.toString().contains(S3_ENCRYPTION_CLIENT_EXCEPTION)) { + return exception; + } + + Throwable cause = exception.getCause(); + if (!(cause instanceof SdkException)) { + return exception; + } + + // get the actual sdk exception. + SdkException sdkCause = (SdkException) cause; + if (sdkCause.getCause() instanceof AwsServiceException) { + return (SdkException) sdkCause.getCause(); + } + + return sdkCause; + } + /** * Translate an exception if it or its inner exception is an * IOException. diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/S3AFileSystemHandler.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/S3AFileSystemOperations.java similarity index 98% rename from hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/S3AFileSystemHandler.java rename to hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/S3AFileSystemOperations.java index 00150ea03fd853..8437d808970407 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/S3AFileSystemHandler.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/S3AFileSystemOperations.java @@ -36,7 +36,7 @@ * An interface that helps map from object store semantics to that of the fileystem. * This specially supports encrypted stores. */ -public interface S3AFileSystemHandler { +public interface S3AFileSystemOperations { /** * Retrieves an object from the S3. diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/S3AStoreImpl.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/S3AStoreImpl.java index 04325634fa3304..db078813455009 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/S3AStoreImpl.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/S3AStoreImpl.java @@ -571,10 +571,10 @@ public Map.Entry deleteObjects( public HeadObjectResponse headObject(String key, ChangeTracker changeTracker, Invoker changeInvoker, - S3AFileSystemHandler fsHandler, + S3AFileSystemOperations fsHandler, String operation) throws IOException { HeadObjectResponse response = getStoreContext().getInvoker() - .retryUntranslated("GET " + key, true, + .retryUntranslated("HEAD " + key, true, () -> { HeadObjectRequest.Builder requestBuilder = getRequestFactory().newHeadObjectRequestBuilder(key); diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AClientSideEncryption.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AClientSideEncryption.java index 1d9958866a368b..5069f949ea221d 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AClientSideEncryption.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AClientSideEncryption.java @@ -18,6 +18,7 @@ package org.apache.hadoop.fs.s3a; +import java.io.FileNotFoundException; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; @@ -33,6 +34,7 @@ import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.LocatedFileStatus; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.RemoteIterator; @@ -61,6 +63,7 @@ import static org.apache.hadoop.fs.s3a.S3ATestUtils.getTestBucketName; import static org.apache.hadoop.fs.s3a.S3ATestUtils.getTestPropertyBool; import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides; +import static org.apache.hadoop.fs.s3a.S3ATestUtils.unsetAllEncryptionPropertiesForBucket; import static org.apache.hadoop.test.LambdaTestUtils.intercept; /** @@ -242,8 +245,8 @@ public void testEncryptionEnabledAndDisabledFS() throws Exception { // CSE enabled FS trying to read unencrypted data would face an exception. try (FSDataInputStream in = cseEnabledFS.open(unEncryptedFilePath)) { - intercept(AWSClientIOException.class, "Instruction file not found!", - "AWSClientIOException should be thrown", + intercept(FileNotFoundException.class, "Instruction file not found!", + "FileNotFoundException should be thrown", () -> { in.read(new byte[SMALL_FILE_SIZE]); return "Exception should be raised if unencrypted data is read by " @@ -273,20 +276,13 @@ public void testEncryptionEnabledAndDisabledFS() throws Exception { /** * Test to check if unencrypted objects are read with V1 client compatibility. - * @throws IOException - * @throws Exception */ @Test public void testUnencryptedObjectReadWithV1CompatibilityConfig() throws Exception { maybeSkipTest(); // initialize base s3 client. Configuration conf = new Configuration(getConfiguration()); - removeBaseAndBucketOverrides(getTestBucketName(conf), - conf, - S3_ENCRYPTION_ALGORITHM, - S3_ENCRYPTION_KEY, - SERVER_SIDE_ENCRYPTION_ALGORITHM, - SERVER_SIDE_ENCRYPTION_KEY); + unsetAllEncryptionPropertiesForBucket(conf); Path file = methodPath(); @@ -314,13 +310,12 @@ public void testUnencryptedObjectReadWithV1CompatibilityConfig() throws Exceptio /** * Tests the size of an encrypted object when with V1 compatibility and custom header length. - * - * @throws Exception If any error occurs during the test execution. */ @Test public void testSizeOfEncryptedObjectFromHeaderWithV1Compatibility() throws Exception { maybeSkipTest(); Configuration cseConf = new Configuration(getConfiguration()); + unsetAllEncryptionPropertiesForBucket(cseConf); cseConf.setBoolean(S3_ENCRYPTION_CSE_V1_COMPATIBILITY_ENABLED, true); try (S3AFileSystem fs = createTestFileSystem(cseConf)) { fs.initialize(getFileSystem().getUri(), cseConf); @@ -346,22 +341,20 @@ public void testSizeOfEncryptedObjectFromHeaderWithV1Compatibility() throws Exce new S3ADataBlocks.BlockUploadData(new byte[SMALL_FILE_SIZE], null), null); - // fetch the random content length - long contentLength = fs.getFileStatus(file).getLen(); - assertEquals("content length does not match", 10, contentLength); + // check if fetched file length matches with the header. + assertFileLength(fs, file, 10); } } } /** * Tests the size of an unencrypted object when using V1 compatibility mode. - * - * @throws Exception If any error occurs during the test execution. */ @Test public void testSizeOfUnencryptedObjectWithV1Compatibility() throws Exception { maybeSkipTest(); Configuration conf = new Configuration(getConfiguration()); + unsetAllEncryptionPropertiesForBucket(conf); conf.setBoolean(S3_ENCRYPTION_CSE_V1_COMPATIBILITY_ENABLED, false); Path file = methodPath(); try (S3AFileSystem fs = createTestFileSystem(conf)) { @@ -372,8 +365,7 @@ public void testSizeOfUnencryptedObjectWithV1Compatibility() throws Exception { SMALL_FILE_SIZE, true); // check the file size - FileStatus status1 = fs.getFileStatus(file); - assertEquals("Mismatch in content length bytes", SMALL_FILE_SIZE, status1.getLen()); + assertFileLength(fs, file, SMALL_FILE_SIZE); } // initialize encrypted s3 client with support for reading unencrypted objects @@ -382,23 +374,19 @@ public void testSizeOfUnencryptedObjectWithV1Compatibility() throws Exception { try (S3AFileSystem cseFs = createTestFileSystem(cseConf)) { cseFs.initialize(getFileSystem().getUri(), cseConf); - // check the file size - FileStatus status2 = cseFs.getFileStatus(file); - assertEquals("Mismatch in content length bytes", SMALL_FILE_SIZE, - status2.getLen()); + assertFileLength(cseFs, file, SMALL_FILE_SIZE); } } /** * Tests the size of an encrypted object when using V1 compatibility mode. - * - * @throws Exception If any error occurs during the test execution. */ @Test public void testSizeOfEncryptedObjectWithV1Compatibility() throws Exception { maybeSkipTest(); Configuration cseConf = new Configuration(getConfiguration()); + unsetAllEncryptionPropertiesForBucket(cseConf); cseConf.setBoolean(S3_ENCRYPTION_CSE_V1_COMPATIBILITY_ENABLED, true); try (S3AFileSystem fs = createTestFileSystem(cseConf)) { fs.initialize(getFileSystem().getUri(), cseConf); @@ -407,10 +395,8 @@ public void testSizeOfEncryptedObjectWithV1Compatibility() throws Exception { Path file = methodPath(); ContractTestUtils.writeDataset(fs, file, new byte[SMALL_FILE_SIZE], SMALL_FILE_SIZE, SMALL_FILE_SIZE, true); - - FileStatus status2 = fs.getFileStatus(file); - assertEquals("Mismatch in content length bytes", - SMALL_FILE_SIZE, status2.getLen()); + // check the file size + assertFileLength(fs, file, SMALL_FILE_SIZE); } } @@ -443,6 +429,24 @@ protected void validateEncryptionForFileSize(int len) throws IOException { rm(getFileSystem(), path, false, false); } + /** + * Asserts that the length of a file in the given FileSystem matches the expected value. + * + *

This method retrieves the FileStatus of the specified file and compares its length + * to the expected value. It uses AssertJ for the assertion, which provides a detailed + * error message if the assertion fails. + * + * @param fs The FileSystem instance containing the file to be checked. + * @param path The Path to the file whose length is to be verified. + * @param expected The expected length of the file in bytes. + */ + private void assertFileLength(FileSystem fs, Path path, long expected) throws IOException { + FileStatus fileStatus = fs.getFileStatus(path); + Assertions.assertThat(fileStatus.getLen()) + .describedAs("Length of %s status: %s", path, fileStatus) + .isEqualTo(expected); + } + /** * Skip tests if certain conditions are met. */ diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AClientSideEncryptionCustom.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AClientSideEncryptionCustom.java index bc7a7b5a5be882..8065ce1b3a759b 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AClientSideEncryptionCustom.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AClientSideEncryptionCustom.java @@ -21,6 +21,8 @@ import java.io.IOException; import java.util.Map; +import org.assertj.core.api.Assertions; + import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.s3a.impl.AWSHeaders; @@ -29,6 +31,7 @@ import static org.apache.hadoop.fs.s3a.Constants.S3_ENCRYPTION_CSE_CUSTOM_KEYRING_CLASS_NAME; import static org.apache.hadoop.fs.s3a.S3ATestUtils.skipIfEncryptionNotSet; import static org.apache.hadoop.fs.s3a.S3ATestUtils.skipIfEncryptionTestsDisabled; +import static org.apache.hadoop.fs.s3a.S3ATestUtils.unsetAllEncryptionPropertiesForBucket; /** * Tests to verify Custom S3 client side encryption CSE-CUSTOM. @@ -45,6 +48,7 @@ public class ITestS3AClientSideEncryptionCustom extends ITestS3AClientSideEncryp protected Configuration createConfiguration() { Configuration conf = super.createConfiguration(); S3ATestUtils.disableFilesystemCaching(conf); + unsetAllEncryptionPropertiesForBucket(conf); conf.set(S3_ENCRYPTION_CSE_CUSTOM_KEYRING_CLASS_NAME, CustomKeyring.class.getCanonicalName()); return conf; @@ -64,9 +68,10 @@ protected void assertEncrypted(Path path) throws IOException { String xAttrPrefix = "header."; // Assert KeyWrap Algo - assertEquals("Key wrap algo isn't same as expected", KMS_KEY_WRAP_ALGO, - processHeader(fsXAttrs, - xAttrPrefix + AWSHeaders.CRYPTO_KEYWRAP_ALGORITHM)); + Assertions.assertThat(processHeader(fsXAttrs, + xAttrPrefix + AWSHeaders.CRYPTO_KEYWRAP_ALGORITHM)) + .describedAs("Key wrap algo") + .isEqualTo(KMS_KEY_WRAP_ALGO); } /** diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java index bfabcb2dff0120..24115177f35a22 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java @@ -646,15 +646,4 @@ private static void skipIfCrossRegionClient( skip("Skipping test as cross region client is in use "); } } - - /** - * Unset encryption options. - * This is needed to avoid encryption tests interfering with non-encryption - * tests. - * @param conf Configuations - */ - private static void unsetEncryption(Configuration conf) { - removeBaseAndBucketOverrides(conf, S3_ENCRYPTION_ALGORITHM); - conf.set(Constants.S3_ENCRYPTION_ALGORITHM, S3AEncryptionMethods.NONE.getMethod()); - } } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ADelayedFNF.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ADelayedFNF.java index bad4d24e4d8a54..ca9d185c3e9e11 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ADelayedFNF.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ADelayedFNF.java @@ -24,7 +24,6 @@ import org.apache.hadoop.fs.contract.ContractTestUtils; import org.apache.hadoop.fs.s3a.impl.ChangeDetectionPolicy; import org.apache.hadoop.fs.s3a.impl.ChangeDetectionPolicy.Source; -import org.apache.hadoop.fs.s3a.impl.CSEUtils; import org.apache.hadoop.test.LambdaTestUtils; import org.junit.Assume; @@ -36,9 +35,7 @@ import static org.apache.hadoop.fs.s3a.Constants.CHANGE_DETECT_SOURCE; import static org.apache.hadoop.fs.s3a.Constants.RETRY_INTERVAL; import static org.apache.hadoop.fs.s3a.Constants.RETRY_LIMIT; -import static org.apache.hadoop.fs.s3a.S3ATestUtils.getTestBucketName; import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides; -import static org.apache.hadoop.fs.s3a.S3AUtils.getEncryptionAlgorithm; /** * Tests behavior of a FileNotFound error that happens after open(), i.e. on @@ -81,14 +78,8 @@ public void testNotFoundFirstRead() throws Exception { final FSDataInputStream in = fs.open(p); assertDeleted(p, false); - Class exceptionClass = FileNotFoundException.class; - if (CSEUtils.isCSEEnabled(getEncryptionAlgorithm( - getTestBucketName(getConfiguration()), getConfiguration()).getMethod())) { - exceptionClass = AWSClientIOException.class; - } - // This should fail since we deleted after the open. - LambdaTestUtils.intercept(exceptionClass, + LambdaTestUtils.intercept(FileNotFoundException.class, () -> in.read()); } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFailureHandling.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFailureHandling.java index be7c25644f9dff..b550fc5864b737 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFailureHandling.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFailureHandling.java @@ -28,7 +28,6 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.RemoteIterator; import org.apache.hadoop.fs.s3a.impl.MultiObjectDeleteException; -import org.apache.hadoop.fs.s3a.impl.CSEUtils; import org.apache.hadoop.fs.statistics.StoreStatisticNames; import org.apache.hadoop.fs.store.audit.AuditSpan; @@ -42,10 +41,8 @@ import static org.apache.hadoop.fs.contract.ContractTestUtils.*; import static org.apache.hadoop.fs.s3a.S3ATestUtils.createFiles; -import static org.apache.hadoop.fs.s3a.S3ATestUtils.getTestBucketName; import static org.apache.hadoop.fs.s3a.S3ATestUtils.isBulkDeleteEnabled; import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides; -import static org.apache.hadoop.fs.s3a.S3AUtils.getEncryptionAlgorithm; import static org.apache.hadoop.fs.s3a.test.ExtraAssertions.failIf; import static org.apache.hadoop.fs.s3a.test.PublicDatasetTestUtils.isUsingDefaultExternalDataFile; import static org.apache.hadoop.fs.s3a.test.PublicDatasetTestUtils.requireDefaultExternalData; @@ -203,14 +200,9 @@ public void testSingleObjectDeleteNoPermissionsTranslated() throws Throwable { Path path = requireDefaultExternalData(getConfiguration()); S3AFileSystem fs = (S3AFileSystem) path.getFileSystem( getConfiguration()); - Class exceptionClass = AccessDeniedException.class; - if (CSEUtils.isCSEEnabled(getEncryptionAlgorithm( - getTestBucketName(getConfiguration()), getConfiguration()).getMethod())) { - exceptionClass = AWSClientIOException.class; - } - Exception ex = (Exception) intercept(exceptionClass, + AccessDeniedException aex = intercept(AccessDeniedException.class, () -> fs.delete(path, false)); - Throwable cause = ex.getCause(); - failIf(cause == null, "no nested exception", ex); + Throwable cause = aex.getCause(); + failIf(cause == null, "no nested exception", aex); } } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java index 3a3f875f5f0d51..66b25a054741eb 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java @@ -635,6 +635,36 @@ public static void print(Logger log, S3ATestUtils.MetricDiff... metrics) { } } + /** + * Unset encryption options. + * @param conf configuration + */ + public static void unsetEncryption(Configuration conf) { + removeBaseAndBucketOverrides(conf, S3_ENCRYPTION_ALGORITHM); + } + + /** + * Removes all encryption-related properties for a specific S3 bucket from given configuration. + * + *

This method unsets various encryption settings specific to the test bucket. It removes + * bucket-specific overrides for multiple encryption-related properties, including both + * client-side and server-side encryption settings. + * + * @param conf The Configuration object from which to remove the encryption properties. + * This object will be modified by this method. + */ + public static void unsetAllEncryptionPropertiesForBucket(Configuration conf) { + removeBucketOverrides(getTestBucketName(conf), + conf, + S3_ENCRYPTION_ALGORITHM, + S3_ENCRYPTION_KEY, + SERVER_SIDE_ENCRYPTION_ALGORITHM, + SERVER_SIDE_ENCRYPTION_KEY, + S3_ENCRYPTION_CSE_CUSTOM_KEYRING_CLASS_NAME, + S3_ENCRYPTION_CSE_V1_COMPATIBILITY_ENABLED, + S3_ENCRYPTION_CSE_KMS_REGION); + } + /** * Print all metrics in a list, then reset them. * @param log log to print the metrics to. diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/ITestConnectionTimeouts.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/ITestConnectionTimeouts.java index cb049cbd2f1ec0..a4cc5cadc5da0f 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/ITestConnectionTimeouts.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/ITestConnectionTimeouts.java @@ -33,7 +33,6 @@ import org.apache.hadoop.fs.FutureDataInputStreamBuilder; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.contract.ContractTestUtils; -import org.apache.hadoop.fs.s3a.AWSClientIOException; import org.apache.hadoop.fs.s3a.AbstractS3ATestBase; import org.apache.hadoop.fs.s3a.S3AFileSystem; import org.apache.hadoop.fs.s3a.api.PerformanceFlagEnum; @@ -60,10 +59,8 @@ import static org.apache.hadoop.fs.s3a.Constants.REQUEST_TIMEOUT; import static org.apache.hadoop.fs.s3a.Constants.RETRY_LIMIT; import static org.apache.hadoop.fs.s3a.Constants.SOCKET_TIMEOUT; -import static org.apache.hadoop.fs.s3a.S3ATestUtils.getTestBucketName; import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides; import static org.apache.hadoop.fs.s3a.commit.CommitConstants.MAGIC_PATH_PREFIX; -import static org.apache.hadoop.fs.s3a.S3AUtils.getEncryptionAlgorithm; import static org.apache.hadoop.fs.s3a.impl.ConfigurationHelper.setDurationAsMillis; import static org.apache.hadoop.test.LambdaTestUtils.intercept; @@ -148,6 +145,7 @@ public void teardown() throws Exception { */ @Test public void testGeneratePoolTimeouts() throws Throwable { + skipIfClientSideEncryption(); AWSClientConfig.setMinimumOperationDuration(Duration.ZERO); Configuration conf = timingOutConfiguration(); Path path = methodPath(); @@ -159,12 +157,7 @@ public void testGeneratePoolTimeouts() throws Throwable { ContractTestUtils.createFile(fs, path, true, DATASET); final FileStatus st = fs.getFileStatus(path); try (FileSystem brittleFS = FileSystem.newInstance(fs.getUri(), conf)) { - Class exceptionClass = ConnectTimeoutException.class; - if (CSEUtils.isCSEEnabled(getEncryptionAlgorithm( - getTestBucketName(conf), conf).getMethod())) { - exceptionClass = AWSClientIOException.class; - } - intercept(exceptionClass, () -> { + intercept(ConnectTimeoutException.class, () -> { for (int i = 0; i < streamsToCreate; i++) { FutureDataInputStreamBuilder b = brittleFS.openFile(path); b.withFileStatus(st); diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestErrorTranslation.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestErrorTranslation.java index 3a4994897a6b9b..a60ffdfab7526f 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestErrorTranslation.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestErrorTranslation.java @@ -30,15 +30,18 @@ import org.junit.Test; import software.amazon.awssdk.awscore.retry.conditions.RetryOnErrorCodeCondition; import software.amazon.awssdk.core.exception.SdkClientException; +import software.amazon.awssdk.core.exception.SdkException; import software.amazon.awssdk.core.retry.RetryPolicyContext; +import software.amazon.awssdk.services.s3.model.NoSuchKeyException; +import software.amazon.encryption.s3.S3EncryptionClientException; import org.apache.hadoop.fs.PathIOException; import org.apache.hadoop.fs.s3a.auth.NoAwsCredentialsException; import org.apache.hadoop.test.AbstractHadoopTestBase; import static org.apache.hadoop.fs.s3a.impl.ErrorTranslation.maybeExtractIOException; +import static org.apache.hadoop.fs.s3a.impl.ErrorTranslation.maybeExtractSdkExceptionFromEncryptionClientException; import static org.apache.hadoop.test.LambdaTestUtils.intercept; -import static org.junit.Assert.assertTrue; /** * Unit tests related to the {@link ErrorTranslation} class. @@ -128,8 +131,34 @@ public void testNoConstructorExtraction() throws Throwable { }); } + @Test + public void testEncryptionClientExceptionExtraction() throws Throwable { + intercept(NoSuchKeyException.class, () -> { + throw maybeExtractSdkExceptionFromEncryptionClientException( + new S3EncryptionClientException("top", + new S3EncryptionClientException("middle", NoSuchKeyException.builder().build()))); + }); + } + + @Test + public void testNonEncryptionClientExceptionExtraction() throws Throwable { + intercept(SdkException.class, () -> { + throw maybeExtractSdkExceptionFromEncryptionClientException( + sdkException("top", sdkException("middle", NoSuchKeyException.builder().build()))); + }); + } + + @Test + public void testEncryptionClientExceptionExtractionWithRTE() throws Throwable { + intercept(S3EncryptionClientException.class, () -> { + throw maybeExtractSdkExceptionFromEncryptionClientException( + new S3EncryptionClientException("top", new UnsupportedOperationException())); + }); + } + + - public static final class NoConstructorIOE extends IOException { + public static final class NoConstructorIOE extends IOException { public static final String MESSAGE = "no-arg constructor"; From d4397bf7a5ef847602ca39776b5957e7daeb883c Mon Sep 17 00:00:00 2001 From: Syed Shameerur Rahman Date: Thu, 7 Nov 2024 19:11:32 +0530 Subject: [PATCH 6/6] PR comments fix --- .../src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java | 4 ++-- .../hadoop/fs/s3a/impl/EncryptionS3ClientFactory.java | 4 ++-- .../org/apache/hadoop/fs/s3a/impl/ErrorTranslation.java | 2 +- .../apache/hadoop/fs/s3a/impl/TestErrorTranslation.java | 8 ++++---- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java index fbc8d39d73a827..a1da63329a9ea9 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java @@ -80,7 +80,7 @@ import static org.apache.hadoop.fs.s3a.Constants.*; import static org.apache.hadoop.fs.s3a.audit.AuditIntegration.maybeTranslateAuditException; import static org.apache.hadoop.fs.s3a.impl.ErrorTranslation.isUnknownBucket; -import static org.apache.hadoop.fs.s3a.impl.ErrorTranslation.maybeExtractSdkExceptionFromEncryptionClientException; +import static org.apache.hadoop.fs.s3a.impl.ErrorTranslation.maybeProcessEncryptionClientException; import static org.apache.hadoop.fs.s3a.impl.InstantiationIOException.instantiationException; import static org.apache.hadoop.fs.s3a.impl.InstantiationIOException.isAbstract; import static org.apache.hadoop.fs.s3a.impl.InstantiationIOException.isNotInstanceOf; @@ -185,7 +185,7 @@ public static IOException translateException(@Nullable String operation, path = "/"; } - exception = maybeExtractSdkExceptionFromEncryptionClientException(exception); + exception = maybeProcessEncryptionClientException(exception); if (!(exception instanceof AwsServiceException)) { // exceptions raised client-side: connectivity, auth, network problems... diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/EncryptionS3ClientFactory.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/EncryptionS3ClientFactory.java index 122c4c2852333f..5817d924febccf 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/EncryptionS3ClientFactory.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/EncryptionS3ClientFactory.java @@ -177,7 +177,7 @@ private S3Client createS3EncryptionClient(S3ClientCreationParameters parameters) keyring = getKeyringProvider(cseMaterials.getCustomKeyringClassName(), cseMaterials.getConf()); } catch (RuntimeException e) { - throw new IOException("Failed to instantiate a custom keyring provider", e); + throw new IOException("Failed to instantiate a custom keyring provider: " + e, e); } CryptographicMaterialsManager customCryptoMaterialsManager = DefaultCryptoMaterialsManager.builder() @@ -260,7 +260,7 @@ private S3AsyncClient createS3AsyncEncryptionClient(S3ClientCreationParameters p keyring = getKeyringProvider(cseMaterials.getCustomKeyringClassName(), cseMaterials.getConf()); } catch (RuntimeException e) { - throw new IOException("Failed to instantiate a custom keyring provider", e); + throw new IOException("Failed to instantiate a custom keyring provider: " + e, e); } CryptographicMaterialsManager customCryptoMaterialsManager = DefaultCryptoMaterialsManager.builder() diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/ErrorTranslation.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/ErrorTranslation.java index eeb13d1025a2b1..cd24b61340c3fe 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/ErrorTranslation.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/ErrorTranslation.java @@ -135,7 +135,7 @@ private static Throwable getInnermostThrowable(Throwable thrown, Throwable outer * @see SdkException * @see AwsServiceException */ - public static SdkException maybeExtractSdkExceptionFromEncryptionClientException( + public static SdkException maybeProcessEncryptionClientException( SdkException exception) { if (exception == null) { return null; diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestErrorTranslation.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestErrorTranslation.java index a60ffdfab7526f..60a3a165e171c5 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestErrorTranslation.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestErrorTranslation.java @@ -40,7 +40,7 @@ import org.apache.hadoop.test.AbstractHadoopTestBase; import static org.apache.hadoop.fs.s3a.impl.ErrorTranslation.maybeExtractIOException; -import static org.apache.hadoop.fs.s3a.impl.ErrorTranslation.maybeExtractSdkExceptionFromEncryptionClientException; +import static org.apache.hadoop.fs.s3a.impl.ErrorTranslation.maybeProcessEncryptionClientException; import static org.apache.hadoop.test.LambdaTestUtils.intercept; /** @@ -134,7 +134,7 @@ public void testNoConstructorExtraction() throws Throwable { @Test public void testEncryptionClientExceptionExtraction() throws Throwable { intercept(NoSuchKeyException.class, () -> { - throw maybeExtractSdkExceptionFromEncryptionClientException( + throw maybeProcessEncryptionClientException( new S3EncryptionClientException("top", new S3EncryptionClientException("middle", NoSuchKeyException.builder().build()))); }); @@ -143,7 +143,7 @@ public void testEncryptionClientExceptionExtraction() throws Throwable { @Test public void testNonEncryptionClientExceptionExtraction() throws Throwable { intercept(SdkException.class, () -> { - throw maybeExtractSdkExceptionFromEncryptionClientException( + throw maybeProcessEncryptionClientException( sdkException("top", sdkException("middle", NoSuchKeyException.builder().build()))); }); } @@ -151,7 +151,7 @@ public void testNonEncryptionClientExceptionExtraction() throws Throwable { @Test public void testEncryptionClientExceptionExtractionWithRTE() throws Throwable { intercept(S3EncryptionClientException.class, () -> { - throw maybeExtractSdkExceptionFromEncryptionClientException( + throw maybeProcessEncryptionClientException( new S3EncryptionClientException("top", new UnsupportedOperationException())); }); }