From e6fb91963278e120dcfdb1ebf5495ae2726df789 Mon Sep 17 00:00:00 2001 From: Steve Loughran Date: Thu, 15 Sep 2016 09:37:05 +0100 Subject: [PATCH 1/7] HADOOP-13614: cut some needless, slower S3A tests --- .../java/org/apache/hadoop/fs/s3a/ITestS3AEncryption.java | 8 +------- .../hadoop/fs/s3a/scale/ITestS3ADeleteManyFiles.java | 7 ------- 2 files changed, 1 insertion(+), 14 deletions(-) diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryption.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryption.java index 454327893f5f93..1fa65237e52278 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryption.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryption.java @@ -48,15 +48,9 @@ protected Configuration createConfiguration() { } private static final int[] SIZES = { - 0, 1, 2, 3, 4, 5, 254, 255, 256, 257, 2 ^ 10 - 3, 2 ^ 11 - 2, 2 ^ 12 - 1 + 0, 1, 2, 3, 4, 5, 254, 255, 256, 257, 2 ^ 12 - 1 }; - @Override - public void teardown() throws Exception { - super.teardown(); - IOUtils.closeStream(getFileSystem()); - } - @Test public void testEncryption() throws Throwable { for (int size: SIZES) { diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3ADeleteManyFiles.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3ADeleteManyFiles.java index 4e1a734e564b99..93eee066fc9c1d 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3ADeleteManyFiles.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3ADeleteManyFiles.java @@ -114,11 +114,4 @@ public Boolean call() throws IOException { ContractTestUtils.assertDeleted(fs, finalDir, true, false); } - @Test - public void testOpenCreate() throws IOException { - final Path scaleTestDir = getTestPath(); - final Path srcDir = new Path(scaleTestDir, "opencreate"); - ContractTestUtils.createAndVerifyFile(fs, srcDir, 1024); - ContractTestUtils.createAndVerifyFile(fs, srcDir, 50 * 1024); - } } From 4b64d18f8baaaa5b994ecf5c70cc430f543617d1 Mon Sep 17 00:00:00 2001 From: Steve Loughran Date: Fri, 16 Sep 2016 14:40:58 +0100 Subject: [PATCH 2/7] HADOOP-13614; fix tests to compile, review all tests, clean slightly and move timing of upload/download times and bandwidth to contract tests --- .../hadoop/fs/contract/ContractTestUtils.java | 6 ++++ .../fs/s3a/ITestS3ABlockingThreadPool.java | 28 +++++-------------- .../hadoop/fs/s3a/ITestS3ABlocksize.java | 3 -- .../hadoop/fs/s3a/ITestS3AConfiguration.java | 6 ++-- ...TestS3AEncryptionAlgorithmPropagation.java | 6 ---- .../scale/ITestS3ADeleteFilesOneByOne.java | 5 ---- 6 files changed, 16 insertions(+), 38 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ContractTestUtils.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ContractTestUtils.java index 73c8f1ce0957e6..f6b6389a102ad3 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ContractTestUtils.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ContractTestUtils.java @@ -834,6 +834,7 @@ public static void verifyReceivedData(FileSystem fs, Path path, long totalBytesRead = 0; int nextExpectedNumber = 0; + NanoTimer timer = new NanoTimer(); try (InputStream inputStream = fs.open(path)) { while (true) { final int bytesRead = inputStream.read(testBuffer); @@ -862,6 +863,8 @@ public static void verifyReceivedData(FileSystem fs, Path path, " bytes but only received " + totalBytesRead); } } + timer.end("Time to read %d bytes", expectedSize); + bandwidth(timer, expectedSize); } /** @@ -925,9 +928,12 @@ public static void createAndVerifyFile(FileSystem fs, Path parent, final long fi final Path objectPath = new Path(parent, objectName); // Write test file in a specific pattern + NanoTimer timer = new NanoTimer(); assertEquals(fileSize, generateTestFile(fs, objectPath, fileSize, testBufferSize, modulus)); assertPathExists(fs, "not created successful", objectPath); + timer.end("Time to write %d bytes", fileSize); + bandwidth(timer, fileSize); // Now read the same file back and verify its content try { diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ABlockingThreadPool.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ABlockingThreadPool.java index 991135e9d7d898..7199c20b0afde7 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ABlockingThreadPool.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ABlockingThreadPool.java @@ -34,32 +34,18 @@ * 4th part should not trigger an exception as it would with a * non-blocking threadpool. */ -public class ITestS3ABlockingThreadPool { - - private Configuration conf; - private S3AFileSystem fs; +public class ITestS3ABlockingThreadPool extends AbstractS3ATestBase { @Rule public Timeout testTimeout = new Timeout(30 * 60 * 1000); - protected Path getTestPath() { - return new Path("/tests3a"); - } - - @Before - public void setUp() throws Exception { - conf = new Configuration(); + @Override + protected Configuration createConfiguration() { + Configuration conf = new Configuration(); conf.setLong(Constants.MIN_MULTIPART_THRESHOLD, 5 * 1024 * 1024); - conf.setLong(Constants.MULTIPART_SIZE, 5 * 1024 * 1024); - conf.setInt(Constants.MAX_THREADS, 2); - conf.setInt(Constants.MAX_TOTAL_TASKS, 1); - } - - @After - public void tearDown() throws Exception { - if (fs != null) { - fs.delete(getTestPath(), true); - } + conf.setInt(Constants.MULTIPART_SIZE, 5 * 1024 * 1024); + conf.setBoolean(Constants.FAST_UPLOAD, true); + return conf; } @Test diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ABlocksize.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ABlocksize.java index 9a6dae75bfdf48..fcd6107135bb22 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ABlocksize.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ABlocksize.java @@ -48,9 +48,6 @@ protected AbstractFSContract createContract(Configuration conf) { return new S3AContract(conf); } - @Rule - public Timeout testTimeout = new Timeout(30 * 60 * 1000); - @Test @SuppressWarnings("deprecation") public void testBlockSize() throws Exception { diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java index 30d4bf66baf53f..773119b9ee4233 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java @@ -76,13 +76,13 @@ public class ITestS3AConfiguration { /** * Test if custom endpoint is picked up. - *

+ *

* The test expects TEST_ENDPOINT to be defined in the Configuration * describing the endpoint of the bucket to which TEST_FS_S3A_NAME points - * (f.i. "s3-eu-west-1.amazonaws.com" if the bucket is located in Ireland). + * (i.e. "s3-eu-west-1.amazonaws.com" if the bucket is located in Ireland). * Evidently, the bucket has to be hosted in the region denoted by the * endpoint for the test to succeed. - *

+ *

* More info and the list of endpoint identifiers: * http://docs.aws.amazon.com/general/latest/gr/rande.html#s3_region * diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionAlgorithmPropagation.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionAlgorithmPropagation.java index 81578c202e891a..c8394ba65b5186 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionAlgorithmPropagation.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionAlgorithmPropagation.java @@ -43,12 +43,6 @@ protected Configuration createConfiguration() { return conf; } - @Override - public void teardown() throws Exception { - super.teardown(); - IOUtils.closeStream(getFileSystem()); - } - @Test public void testEncrypt0() throws Throwable { writeThenReadFileToFailure(0); diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3ADeleteFilesOneByOne.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3ADeleteFilesOneByOne.java index a375664a50ac48..3e2a5b8574b25a 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3ADeleteFilesOneByOne.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3ADeleteFilesOneByOne.java @@ -36,9 +36,4 @@ protected Configuration createConfiguration() { return configuration; } - @Override - @Test - public void testOpenCreate() throws IOException { - - } } From db246a8ab26eb73380f8523607971b9809702e9d Mon Sep 17 00:00:00 2001 From: Steve Loughran Date: Fri, 30 Sep 2016 18:28:14 +0100 Subject: [PATCH 3/7] HADOOP-13614: patch 002. -where possible give all the s3a tests the superclass of {{AbstractS3ATestBase}}, clean up children. -ITestS3AFileSystemContract has path logic reworked *so test can run in parallel* -fixed up yarn test to work in parallel too, along with BlockingThreadPool and FastOutputStream -on the files changed, moved the copyright comment to being a simple comment rather than a javadoc one. This conflicts a bit with HADOOP-13560; this patch should go in first and then I'll address the diffs --- .../TestFSMainOperationsLocalFileSystem.java | 4 +-- hadoop-tools/hadoop-aws/pom.xml | 7 ---- .../hadoop/fs/contract/s3a/S3AContract.java | 6 ++-- .../hadoop/fs/s3a/AbstractS3ATestBase.java | 3 -- .../fs/s3a/ITestS3ABlockingThreadPool.java | 18 ++-------- .../hadoop/fs/s3a/ITestS3ABlocksize.java | 4 +-- .../hadoop/fs/s3a/ITestS3AEncryption.java | 1 - ...TestS3AEncryptionAlgorithmPropagation.java | 1 - .../fs/s3a/ITestS3AFailureHandling.java | 9 ++--- .../fs/s3a/ITestS3AFileOperationCost.java | 16 +-------- .../fs/s3a/ITestS3AFileSystemContract.java | 33 +++++++++++++++++-- .../fs/s3a/ITestS3ATemporaryCredentials.java | 14 ++------ .../hadoop/fs/s3a/S3ATestConstants.java | 8 +++++ .../apache/hadoop/fs/s3a/S3ATestUtils.java | 14 ++++++++ .../fs/s3a/scale/ITestS3ADeleteManyFiles.java | 2 +- .../scale/ITestS3ADirectoryPerformance.java | 8 ++--- .../scale/ITestS3AInputStreamPerformance.java | 2 +- .../hadoop/fs/s3a/scale/S3AScaleTestBase.java | 27 +++++++-------- .../apache/hadoop/fs/s3a/yarn/ITestS3A.java | 4 +-- .../fs/s3a/yarn/ITestS3AMiniYarnCluster.java | 32 +++++++++--------- 20 files changed, 102 insertions(+), 111 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestFSMainOperationsLocalFileSystem.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestFSMainOperationsLocalFileSystem.java index 6081f38570699d..12687fd8b9289d 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestFSMainOperationsLocalFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestFSMainOperationsLocalFileSystem.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -46,7 +46,7 @@ public void setUp() throws Exception { fcTarget = FileSystem.getLocal(conf); super.setUp(); } - + @Override @After public void tearDown() throws Exception { diff --git a/hadoop-tools/hadoop-aws/pom.xml b/hadoop-tools/hadoop-aws/pom.xml index 28825d940b2a2b..1c70638dea2d43 100644 --- a/hadoop-tools/hadoop-aws/pom.xml +++ b/hadoop-tools/hadoop-aws/pom.xml @@ -181,9 +181,6 @@ **/ITestJets3tNativeS3FileSystemContract.java - **/ITestS3ABlockingThreadPool.java - **/ITestS3AFileSystemContract.java - **/ITestS3AMiniYarnCluster.java **/ITest*Root*.java **/ITestS3AFileContextStatistics.java **/ITestS3AHuge*.java @@ -211,10 +208,6 @@ **/ITestJets3tNativeS3FileSystemContract.java - **/ITestS3ABlockingThreadPool.java - **/ITestS3AFastOutputStream.java - **/ITestS3AFileSystemContract.java - **/ITestS3AMiniYarnCluster.java **/ITest*Root*.java **/ITestS3AFileContextStatistics.java **/ITestS3AHuge*.java diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/S3AContract.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/S3AContract.java index e9024b5e822946..3510a6486a2609 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/S3AContract.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/S3AContract.java @@ -21,6 +21,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.contract.AbstractBondedFSContract; +import org.apache.hadoop.fs.s3a.S3ATestUtils; /** * The contract of S3A: only enabled if the test bucket is provided. @@ -29,7 +30,6 @@ public class S3AContract extends AbstractBondedFSContract { public static final String CONTRACT_XML = "contract/s3a.xml"; - public S3AContract(Configuration conf) { super(conf); //insert the base features @@ -43,8 +43,6 @@ public String getScheme() { @Override public Path getTestPath() { - String testUniqueForkId = System.getProperty("test.unique.fork.id"); - return testUniqueForkId == null ? super.getTestPath() : - new Path("/" + testUniqueForkId, "test"); + return S3ATestUtils.createTestPath(super.getTestPath()); } } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractS3ATestBase.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractS3ATestBase.java index e049fd16aa328e..358426d95636d7 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractS3ATestBase.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractS3ATestBase.java @@ -52,9 +52,6 @@ public void teardown() throws Exception { IOUtils.closeStream(getFileSystem()); } - @Rule - public TestName methodName = new TestName(); - @Before public void nameThread() { Thread.currentThread().setName("JUnit-" + methodName.getMethodName()); diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ABlockingThreadPool.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ABlockingThreadPool.java index 7199c20b0afde7..b4b6ce589eaa49 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ABlockingThreadPool.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ABlockingThreadPool.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -19,13 +19,8 @@ package org.apache.hadoop.fs.s3a; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.contract.ContractTestUtils; -import org.junit.After; -import org.junit.Before; -import org.junit.Rule; import org.junit.Test; -import org.junit.rules.Timeout; /** * Demonstrate that the threadpool blocks additional client requests if @@ -48,21 +43,14 @@ protected Configuration createConfiguration() { return conf; } - @Test - public void testRegularMultiPartUpload() throws Exception { - fs = S3ATestUtils.createTestFileSystem(conf); - ContractTestUtils.createAndVerifyFile(fs, getTestPath(), 16 * 1024 * - 1024); - } - @Test public void testFastMultiPartUpload() throws Exception { conf.setBoolean(Constants.FAST_UPLOAD, true); conf.set(Constants.FAST_UPLOAD_BUFFER, Constants.FAST_UPLOAD_BYTEBUFFER); fs = S3ATestUtils.createTestFileSystem(conf); - ContractTestUtils.createAndVerifyFile(fs, getTestPath(), 16 * 1024 * - 1024); + ContractTestUtils.createAndVerifyFile(fs, + path("testFastMultiPartUpload"), 16 * 1024 * 1024); } } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ABlocksize.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ABlocksize.java index fcd6107135bb22..e8a4e40b182117 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ABlocksize.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ABlocksize.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -25,9 +25,7 @@ import org.apache.hadoop.fs.contract.AbstractFSContract; import org.apache.hadoop.fs.contract.AbstractFSContractTestBase; import org.apache.hadoop.fs.contract.s3a.S3AContract; -import org.junit.Rule; import org.junit.Test; -import org.junit.rules.Timeout; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryption.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryption.java index 1fa65237e52278..8432789979e723 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryption.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryption.java @@ -22,7 +22,6 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.contract.ContractTestUtils; -import org.apache.hadoop.io.IOUtils; import org.junit.Test; import java.io.IOException; diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionAlgorithmPropagation.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionAlgorithmPropagation.java index c8394ba65b5186..96deb2558642e8 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionAlgorithmPropagation.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionAlgorithmPropagation.java @@ -20,7 +20,6 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.io.IOUtils; import org.junit.Test; import java.io.IOException; diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFailureHandling.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFailureHandling.java index e284ea7226da2c..8116c2c80863d7 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFailureHandling.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFailureHandling.java @@ -18,7 +18,6 @@ package org.apache.hadoop.fs.s3a; -import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -36,20 +35,16 @@ import java.util.concurrent.Callable; import static org.apache.hadoop.fs.contract.ContractTestUtils.*; +import static org.apache.hadoop.fs.s3a.S3ATestUtils.*; /** * Test S3A Failure translation, including a functional test * generating errors during stream IO. */ -public class ITestS3AFailureHandling extends AbstractFSContractTestBase { +public class ITestS3AFailureHandling extends AbstractS3ATestBase { private static final Logger LOG = LoggerFactory.getLogger(ITestS3AFailureHandling.class); - @Override - protected AbstractFSContract createContract(Configuration conf) { - return new S3AContract(conf); - } - @Test public void testReadFileChanged() throws Throwable { describe("overwrite a file with a shorter one during a read, seek"); diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileOperationCost.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileOperationCost.java index f19ea954bc54c1..8616e263ce0374 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileOperationCost.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileOperationCost.java @@ -18,13 +18,9 @@ package org.apache.hadoop.fs.s3a; -import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.contract.AbstractFSContract; -import org.apache.hadoop.fs.contract.AbstractFSContractTestBase; import org.apache.hadoop.fs.contract.ContractTestUtils; -import org.apache.hadoop.fs.contract.s3a.S3AContract; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -43,7 +39,7 @@ * Use metrics to assert about the cost of file status queries. * {@link S3AFileSystem#getFileStatus(Path)}. */ -public class ITestS3AFileOperationCost extends AbstractFSContractTestBase { +public class ITestS3AFileOperationCost extends AbstractS3ATestBase { private MetricDiff metadataRequests; private MetricDiff listRequests; @@ -51,16 +47,6 @@ public class ITestS3AFileOperationCost extends AbstractFSContractTestBase { private static final Logger LOG = LoggerFactory.getLogger(ITestS3AFileOperationCost.class); - @Override - protected AbstractFSContract createContract(Configuration conf) { - return new S3AContract(conf); - } - - @Override - public S3AFileSystem getFileSystem() { - return (S3AFileSystem) super.getFileSystem(); - } - @Override public void setup() throws Exception { super.setup(); diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileSystemContract.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileSystemContract.java index 858ac22c7f216b..619d09cccc64aa 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileSystemContract.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileSystemContract.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -18,6 +18,9 @@ package org.apache.hadoop.fs.s3a; +import org.junit.Before; +import org.junit.Rule; +import org.junit.rules.TestName; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; @@ -38,18 +41,44 @@ public class ITestS3AFileSystemContract extends FileSystemContractBaseTest { protected static final Logger LOG = LoggerFactory.getLogger(ITestS3AFileSystemContract.class); + private Path basePath; + + @Rule + public TestName methodName = new TestName(); + + @Before + public void nameThread() { + Thread.currentThread().setName("JUnit-" + methodName.getMethodName()); + } + @Override public void setUp() throws Exception { Configuration conf = new Configuration(); fs = S3ATestUtils.createTestFileSystem(conf); + basePath = fs.makeQualified( + S3ATestUtils.createTestPath(new Path("/s3afilesystemcontract"))); super.setUp(); } + /** + * This path explicitly places all absolute paths under the per-test suite + * path directory; this allows the test to run in parallel. + * @param pathString path string as input + * @return a qualified path string. + */ + protected Path path(String pathString) { + if (pathString.startsWith("/")) { + return new Path(basePath, pathString).makeQualified(fs); + } else { + return super.path(pathString); + } + } + @Override protected void tearDown() throws Exception { if (fs != null) { - fs.delete(path("test"), true); + fs.delete(basePath, true); } super.tearDown(); } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ATemporaryCredentials.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ATemporaryCredentials.java index 360a151406a320..84aad3c9b7ffe5 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ATemporaryCredentials.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ATemporaryCredentials.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -28,9 +28,6 @@ import com.amazonaws.services.securitytoken.model.GetSessionTokenResult; import com.amazonaws.services.securitytoken.model.Credentials; -import org.apache.hadoop.fs.contract.AbstractFSContract; -import org.apache.hadoop.fs.contract.AbstractFSContractTestBase; -import org.apache.hadoop.fs.contract.s3a.S3AContract; import org.apache.hadoop.fs.s3native.S3xLoginHelper; import org.apache.hadoop.conf.Configuration; @@ -48,9 +45,7 @@ * should only be used against transient filesystems where you don't care about * the data. */ -public class ITestS3ATemporaryCredentials extends AbstractFSContractTestBase { - public static final String TEST_STS_ENABLED = "test.fs.s3a.sts.enabled"; - public static final String TEST_STS_ENDPOINT = "test.fs.s3a.sts.endpoint"; +public class ITestS3ATemporaryCredentials extends AbstractS3ATestBase { private static final Logger LOG = LoggerFactory.getLogger(ITestS3ATemporaryCredentials.class); @@ -60,11 +55,6 @@ public class ITestS3ATemporaryCredentials extends AbstractFSContractTestBase { private static final long TEST_FILE_SIZE = 1024; - @Override - protected AbstractFSContract createContract(Configuration conf) { - return new S3AContract(conf); - } - /** * Test use of STS for requesting temporary credentials. * diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestConstants.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestConstants.java index 6894bb07075de3..e62c47bdb3ef43 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestConstants.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestConstants.java @@ -144,4 +144,12 @@ public interface S3ATestConstants { */ boolean DEFAULT_SCALE_TESTS_ENABLED = false; + String KEY_ENCRYPTION_TESTS = TEST_FS_S3A + "encryption.enabled"; + + /** + * Fork ID passed down from maven if the test is running in parallel. + */ + String TEST_UNIQUE_FORK_ID = "test.unique.fork.id"; + String TEST_STS_ENABLED = "test.fs.s3a.sts.enabled"; + String TEST_STS_ENDPOINT = "test.fs.s3a.sts.endpoint"; } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java index 19dccac15b3a2a..5d52b3722ef2a1 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java @@ -21,6 +21,7 @@ import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileContext; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.s3a.scale.S3AScaleTestBase; import org.junit.Assert; import org.junit.internal.AssumptionViolatedException; @@ -301,6 +302,19 @@ public static void skipIfEncryptionTestsDisabled( } } + /** + * Create a test path, using the value of + * {@link S3ATestConstants#TEST_UNIQUE_FORK_ID} if it is set. + * @param defVal default value + * @return a path + */ + public static Path createTestPath(Path defVal) { + String testUniqueForkId = System.getProperty( + S3ATestConstants.TEST_UNIQUE_FORK_ID); + return testUniqueForkId == null ? defVal : + new Path("/" + testUniqueForkId, "test"); + } + /** * Reset all metrics in a list. * @param metrics metrics to reset diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3ADeleteManyFiles.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3ADeleteManyFiles.java index 93eee066fc9c1d..fbbc58f8aa68e3 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3ADeleteManyFiles.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3ADeleteManyFiles.java @@ -50,7 +50,7 @@ public class ITestS3ADeleteManyFiles extends S3AScaleTestBase { */ @Test public void testBulkRenameAndDelete() throws Throwable { - final Path scaleTestDir = getTestPath(); + final Path scaleTestDir = path("testBulkRenameAndDelete"); final Path srcDir = new Path(scaleTestDir, "src"); final Path finalDir = new Path(scaleTestDir, "final"); final long count = getOperationCount(); diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3ADirectoryPerformance.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3ADirectoryPerformance.java index b5f4eb3cc8300d..1d0f9e1c3a7d6b 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3ADirectoryPerformance.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3ADirectoryPerformance.java @@ -40,7 +40,7 @@ public class ITestS3ADirectoryPerformance extends S3AScaleTestBase { @Test public void testListOperations() throws Throwable { describe("Test recursive list operations"); - final Path scaleTestDir = getTestPath(); + final Path scaleTestDir = path("testListOperations"); final Path listDir = new Path(scaleTestDir, "lists"); // scale factor. @@ -137,7 +137,7 @@ public void testListOperations() throws Throwable { @Test public void testTimeToStatEmptyDirectory() throws Throwable { describe("Time to stat an empty directory"); - Path path = new Path(getTestPath(), "empty"); + Path path = path("empty"); fs.mkdirs(path); timeToStatPath(path); } @@ -145,7 +145,7 @@ public void testTimeToStatEmptyDirectory() throws Throwable { @Test public void testTimeToStatNonEmptyDirectory() throws Throwable { describe("Time to stat a non-empty directory"); - Path path = new Path(getTestPath(), "dir"); + Path path = path("dir"); fs.mkdirs(path); touch(fs, new Path(path, "file")); timeToStatPath(path); @@ -154,7 +154,7 @@ public void testTimeToStatNonEmptyDirectory() throws Throwable { @Test public void testTimeToStatFile() throws Throwable { describe("Time to stat a simple file"); - Path path = new Path(getTestPath(), "file"); + Path path = path("file"); touch(fs, path); timeToStatPath(path); } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3AInputStreamPerformance.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3AInputStreamPerformance.java index e2163c5da94c26..090acc6c3482dd 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3AInputStreamPerformance.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3AInputStreamPerformance.java @@ -436,7 +436,7 @@ public void testRandomReadOverBuffer() throws Throwable { describe("read over a buffer, making sure that the requests" + " spans readahead ranges"); int datasetLen = _32K; - Path dataFile = new Path(getTestPath(), "testReadOverBuffer.bin"); + Path dataFile = path("testReadOverBuffer.bin"); byte[] sourceData = dataset(datasetLen, 0, 64); // relies on the field 'fs' referring to the R/W FS writeDataset(fs, dataFile, sourceData, datasetLen, _16K, true); diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/S3AScaleTestBase.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/S3AScaleTestBase.java index af6d468fb64b22..7a929f3d2a0cd7 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/S3AScaleTestBase.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/S3AScaleTestBase.java @@ -22,6 +22,7 @@ import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.contract.ContractTestUtils; +import org.apache.hadoop.fs.s3a.AbstractS3ATestBase; import org.apache.hadoop.fs.s3a.S3AFileSystem; import org.apache.hadoop.fs.s3a.S3AInputStream; import org.apache.hadoop.fs.s3a.S3AInstrumentation; @@ -35,6 +36,7 @@ import org.junit.Rule; import org.junit.rules.TestName; import org.junit.rules.Timeout; +import org.junit.Assert; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -46,19 +48,11 @@ * Base class for scale tests; here is where the common scale configuration * keys are defined. */ -public class S3AScaleTestBase extends Assert implements S3ATestConstants { - - @Rule - public final TestName methodName = new TestName(); +public class S3AScaleTestBase extends AbstractS3ATestBase { @Rule public Timeout testTimeout = createTestTimeout(); - @Before - public void nameThread() { - Thread.currentThread().setName("JUnit"); - } - public static final int _1KB = 1024; public static final int _1MB = _1KB * _1KB; @@ -79,6 +73,7 @@ public void nameThread() { protected Configuration createConfiguration() { return new Configuration(); } + private Path testPath; /** * Get the configuration used to set up the FS. @@ -114,18 +109,18 @@ private synchronized Configuration demandCreateConfiguration() { if (conf == null) { conf = createConfiguration(); } - return conf; + return getConfiguration(); } - @After - public void tearDown() throws Exception { - ContractTestUtils.rm(fs, getTestPath(), true, true); + @Override + public void setup() throws Exception { + super.setup(); + testPath = path("/tests3a"); + fs = getFileSystem(); } protected Path getTestPath() { - String testUniqueForkId = System.getProperty("test.unique.fork.id"); - return testUniqueForkId == null ? new Path("/tests3a") : - new Path("/" + testUniqueForkId, "tests3a"); + return testPath; } protected long getOperationCount() { diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/yarn/ITestS3A.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/yarn/ITestS3A.java index ca57da65f3cc95..7d2c1dc302377b 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/yarn/ITestS3A.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/yarn/ITestS3A.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -59,7 +59,7 @@ public void tearDown() throws Exception { } protected Path getTestPath() { - return new Path("/tests3afc"); + return S3ATestUtils.createTestPath(new Path("/tests3afc")); } @Test diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/yarn/ITestS3AMiniYarnCluster.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/yarn/ITestS3AMiniYarnCluster.java index 772d8c7fd0beed..ac8caf8e79ab8c 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/yarn/ITestS3AMiniYarnCluster.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/yarn/ITestS3AMiniYarnCluster.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with this * work for additional information regarding copyright ownership. The ASF @@ -28,6 +28,7 @@ import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileContext; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.s3a.AbstractS3ATestBase; import org.apache.hadoop.fs.s3a.S3AFileSystem; import org.apache.hadoop.fs.s3a.S3ATestUtils; import org.apache.hadoop.io.IOUtils; @@ -39,26 +40,26 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.MiniYARNCluster; -import org.junit.After; -import static org.junit.Assert.assertTrue; -import org.junit.Before; import org.junit.Test; -import static org.junit.Assert.assertEquals; /** * Tests that S3A is usable through a YARN application. */ -public class ITestS3AMiniYarnCluster { +public class ITestS3AMiniYarnCluster extends AbstractS3ATestBase { private final Configuration conf = new YarnConfiguration(); private S3AFileSystem fs; private MiniYARNCluster yarnCluster; - private final String rootPath = "/tests/MiniClusterWordCount/"; + private Path rootPath; - @Before - public void beforeTest() throws IOException { + @Override + public void setup() throws Exception { + super.setup(); fs = S3ATestUtils.createTestFileSystem(conf); - fs.mkdirs(new Path(rootPath + "input/")); + rootPath = path("MiniClusterWordCount"); + Path workingDir = path("working"); + fs.setWorkingDirectory(workingDir); + fs.mkdirs(new Path(rootPath, "input/")); yarnCluster = new MiniYARNCluster("MiniClusterWordCount", // testName 1, // number of node managers @@ -68,17 +69,18 @@ public void beforeTest() throws IOException { yarnCluster.start(); } - @After - public void afterTest() throws IOException { - fs.delete(new Path(rootPath), true); + @Override + public void teardown() throws Exception { + fs.delete(rootPath, true); yarnCluster.stop(); + super.teardown(); } @Test public void testWithMiniCluster() throws Exception { - Path input = new Path(rootPath + "input/in.txt"); + Path input = new Path(rootPath, "input/in.txt"); input = input.makeQualified(fs.getUri(), fs.getWorkingDirectory()); - Path output = new Path(rootPath + "output/"); + Path output = new Path(rootPath, "output/"); output = output.makeQualified(fs.getUri(), fs.getWorkingDirectory()); writeStringToFile(input, "first line\nsecond line\nthird line"); From 6c50eb316c93ccab27876f2065d50ea4bb1064a5 Mon Sep 17 00:00:00 2001 From: Steve Loughran Date: Tue, 4 Oct 2016 16:19:06 +0100 Subject: [PATCH 4/7] HADOOP-13614 patch 004: tuning timeouts with 10 min for default tests, 30 min for the long tests --- .../hadoop/fs/contract/s3a/ITestS3AContractDistCp.java | 6 ++++++ .../org/apache/hadoop/fs/s3a/AbstractS3ATestBase.java | 7 +++++-- .../org/apache/hadoop/fs/s3a/S3ATestConstants.java | 10 ++++++++++ 3 files changed, 21 insertions(+), 2 deletions(-) diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractDistCp.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractDistCp.java index 9e14ed2d2cee99..2e92c7f276c99a 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractDistCp.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractDistCp.java @@ -19,6 +19,7 @@ package org.apache.hadoop.fs.contract.s3a; import static org.apache.hadoop.fs.s3a.Constants.*; +import static org.apache.hadoop.fs.s3a.S3ATestConstants.SCALE_TEST_TIMEOUT; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.tools.contract.AbstractContractDistCpTest; @@ -32,6 +33,11 @@ public class ITestS3AContractDistCp extends AbstractContractDistCpTest { private static final long MULTIPART_SETTING = MULTIPART_MIN_SIZE; + @Override + protected int getTestTimeoutMillis() { + return SCALE_TEST_TIMEOUT; + } + @Override protected Configuration createConfiguration() { Configuration newConf = super.createConfiguration(); diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractS3ATestBase.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractS3ATestBase.java index 358426d95636d7..c793982417bdc2 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractS3ATestBase.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractS3ATestBase.java @@ -26,8 +26,6 @@ import org.apache.hadoop.fs.contract.s3a.S3AContract; import org.apache.hadoop.io.IOUtils; import org.junit.Before; -import org.junit.Rule; -import org.junit.rules.TestName; import java.io.IOException; @@ -57,6 +55,11 @@ public void nameThread() { Thread.currentThread().setName("JUnit-" + methodName.getMethodName()); } + @Override + protected int getTestTimeoutMillis() { + return S3A_TEST_TIMEOUT; + } + protected Configuration getConfiguration() { return getContract().getConf(); } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestConstants.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestConstants.java index e62c47bdb3ef43..daaf948f9ada36 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestConstants.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestConstants.java @@ -152,4 +152,14 @@ public interface S3ATestConstants { String TEST_UNIQUE_FORK_ID = "test.unique.fork.id"; String TEST_STS_ENABLED = "test.fs.s3a.sts.enabled"; String TEST_STS_ENDPOINT = "test.fs.s3a.sts.endpoint"; + + /** + * Timeout in Milliseconds for standard tests: {@value}. + */ + int S3A_TEST_TIMEOUT = 10 * 60 * 1000; + + /** + * Timeout in Milliseconds for Scale Tests: {@value}. + */ + int SCALE_TEST_TIMEOUT = 30 * 60 * 1000; } From fbbea164f8edb1c9d2a14c1544e0f32b2a92a95e Mon Sep 17 00:00:00 2001 From: Steve Loughran Date: Thu, 6 Oct 2016 13:43:16 +0100 Subject: [PATCH 5/7] HADOOP-13614 checkstyle/javac cleanup --- .../org/apache/hadoop/fs/s3a/ITestS3AFileSystemContract.java | 2 +- .../hadoop/fs/s3a/scale/ITestS3ADeleteFilesOneByOne.java | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileSystemContract.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileSystemContract.java index 619d09cccc64aa..0eb601bdb44611 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileSystemContract.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileSystemContract.java @@ -69,7 +69,7 @@ public void setUp() throws Exception { */ protected Path path(String pathString) { if (pathString.startsWith("/")) { - return new Path(basePath, pathString).makeQualified(fs); + return fs.makeQualified(new Path(basePath, pathString)); } else { return super.path(pathString); } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3ADeleteFilesOneByOne.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3ADeleteFilesOneByOne.java index 3e2a5b8574b25a..b054f163005116 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3ADeleteFilesOneByOne.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3ADeleteFilesOneByOne.java @@ -20,9 +20,6 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.s3a.Constants; -import org.junit.Test; - -import java.io.IOException; /** * Tests file deletion with multi-delete disabled. From 20c1669f247dc2e5b6657763df7000ee8cbbace3 Mon Sep 17 00:00:00 2001 From: Steve Loughran Date: Wed, 19 Oct 2016 14:06:35 +0100 Subject: [PATCH 6/7] HADOOP-13614 apply to branch-2; test and verify that all appears well. Tune more tests, including ITestS3AMiniYarnCluster. A big change here is fixing up the scale tests to work as subclasses of the S3AScaleTestBase indirect subclasses of AbstractFSContractTestBase, because that sets up the test timeout rule. Rather than have a field of the same name and hope that its timeout gets picked up, I've tuned how timeouts get set up, so the subclasses do it. All well and good, except those subclasses are being called during the initialization of the base class, that is: before the subclasses are full inited. I don't ever like doing that, though it is working here. --- .../contract/s3a/ITestS3AContractDistCp.java | 4 +- .../hadoop/fs/s3a/AbstractS3ATestBase.java | 16 +++ .../fs/s3a/ITestS3ABlockingThreadPool.java | 56 -------- .../hadoop/fs/s3a/ITestS3ABlocksize.java | 10 +- .../hadoop/fs/s3a/ITestS3AConfiguration.java | 17 ++- .../fs/s3a/ITestS3AFileOperationCost.java | 3 +- .../hadoop/fs/s3a/S3ATestConstants.java | 19 +-- .../apache/hadoop/fs/s3a/S3ATestUtils.java | 2 +- .../s3a/scale/AbstractSTestS3AHugeFiles.java | 25 ++-- .../scale/ITestS3ADeleteFilesOneByOne.java | 4 +- .../fs/s3a/scale/ITestS3ADeleteManyFiles.java | 4 +- .../scale/ITestS3ADirectoryPerformance.java | 8 +- .../scale/ITestS3AHugeFilesClassicOutput.java | 4 +- .../scale/ITestS3AInputStreamPerformance.java | 1 + .../hadoop/fs/s3a/scale/S3AScaleTestBase.java | 128 +++++++++--------- .../fs/s3a/yarn/ITestS3AMiniYarnCluster.java | 18 +-- 16 files changed, 144 insertions(+), 175 deletions(-) delete mode 100644 hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ABlockingThreadPool.java diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractDistCp.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractDistCp.java index 2e92c7f276c99a..50ce0c2a98f401 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractDistCp.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractDistCp.java @@ -19,7 +19,7 @@ package org.apache.hadoop.fs.contract.s3a; import static org.apache.hadoop.fs.s3a.Constants.*; -import static org.apache.hadoop.fs.s3a.S3ATestConstants.SCALE_TEST_TIMEOUT; +import static org.apache.hadoop.fs.s3a.S3ATestConstants.SCALE_TEST_TIMEOUT_MILLIS; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.tools.contract.AbstractContractDistCpTest; @@ -35,7 +35,7 @@ public class ITestS3AContractDistCp extends AbstractContractDistCpTest { @Override protected int getTestTimeoutMillis() { - return SCALE_TEST_TIMEOUT; + return SCALE_TEST_TIMEOUT_MILLIS; } @Override diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractS3ATestBase.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractS3ATestBase.java index c793982417bdc2..c19b72cde9fcce 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractS3ATestBase.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractS3ATestBase.java @@ -26,6 +26,8 @@ import org.apache.hadoop.fs.contract.s3a.S3AContract; import org.apache.hadoop.io.IOUtils; import org.junit.Before; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.IOException; @@ -38,6 +40,9 @@ public abstract class AbstractS3ATestBase extends AbstractFSContractTestBase implements S3ATestConstants { + protected static final Logger LOG = + LoggerFactory.getLogger(AbstractS3ATestBase.class); + @Override protected AbstractFSContract createContract(Configuration conf) { return new S3AContract(conf); @@ -73,6 +78,17 @@ public S3AFileSystem getFileSystem() { return (S3AFileSystem) super.getFileSystem(); } + /** + * Describe a test in the logs. + * @param text text to print + * @param args arguments to format in the printing + */ + protected void describe(String text, Object... args) { + LOG.info("\n\n{}: {}\n", + methodName.getMethodName(), + String.format(text, args)); + } + /** * Write a file, read it back, validate the dataset. Overwrites the file * if it is present diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ABlockingThreadPool.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ABlockingThreadPool.java deleted file mode 100644 index b4b6ce589eaa49..00000000000000 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ABlockingThreadPool.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.fs.s3a; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.contract.ContractTestUtils; -import org.junit.Test; - -/** - * Demonstrate that the threadpool blocks additional client requests if - * its queue is full (rather than throwing an exception) by initiating an - * upload consisting of 4 parts with 2 threads and 1 spot in the queue. The - * 4th part should not trigger an exception as it would with a - * non-blocking threadpool. - */ -public class ITestS3ABlockingThreadPool extends AbstractS3ATestBase { - - @Rule - public Timeout testTimeout = new Timeout(30 * 60 * 1000); - - @Override - protected Configuration createConfiguration() { - Configuration conf = new Configuration(); - conf.setLong(Constants.MIN_MULTIPART_THRESHOLD, 5 * 1024 * 1024); - conf.setInt(Constants.MULTIPART_SIZE, 5 * 1024 * 1024); - conf.setBoolean(Constants.FAST_UPLOAD, true); - return conf; - } - - @Test - public void testFastMultiPartUpload() throws Exception { - conf.setBoolean(Constants.FAST_UPLOAD, true); - conf.set(Constants.FAST_UPLOAD_BUFFER, - Constants.FAST_UPLOAD_BYTEBUFFER); - fs = S3ATestUtils.createTestFileSystem(conf); - ContractTestUtils.createAndVerifyFile(fs, - path("testFastMultiPartUpload"), 16 * 1024 * 1024); - - } -} diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ABlocksize.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ABlocksize.java index e8a4e40b182117..30fb53439f325e 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ABlocksize.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ABlocksize.java @@ -25,7 +25,10 @@ import org.apache.hadoop.fs.contract.AbstractFSContract; import org.apache.hadoop.fs.contract.AbstractFSContractTestBase; import org.apache.hadoop.fs.contract.s3a.S3AContract; + +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.Timeout; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -36,16 +39,11 @@ /** * S3A tests for configuring block size. */ -public class ITestS3ABlocksize extends AbstractFSContractTestBase { +public class ITestS3ABlocksize extends AbstractS3ATestBase { private static final Logger LOG = LoggerFactory.getLogger(ITestS3ABlocksize.class); - @Override - protected AbstractFSContract createContract(Configuration conf) { - return new S3AContract(conf); - } - @Test @SuppressWarnings("deprecation") public void testBlockSize() throws Exception { diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java index 773119b9ee4233..68168344d38ea0 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java @@ -66,10 +66,10 @@ public class ITestS3AConfiguration { private static final Logger LOG = LoggerFactory.getLogger(ITestS3AConfiguration.class); - private static final String TEST_ENDPOINT = "test.fs.s3a.endpoint"; - @Rule - public Timeout testTimeout = new Timeout(30 * 60 * 1000); + public Timeout testTimeout = new Timeout( + S3ATestConstants.S3A_TEST_TIMEOUT + ); @Rule public final TemporaryFolder tempDir = new TemporaryFolder(); @@ -77,23 +77,26 @@ public class ITestS3AConfiguration { /** * Test if custom endpoint is picked up. *

- * The test expects TEST_ENDPOINT to be defined in the Configuration + * The test expects {@link S3ATestConstants#CONFIGURATION_TEST_ENDPOINT} + * to be defined in the Configuration * describing the endpoint of the bucket to which TEST_FS_S3A_NAME points * (i.e. "s3-eu-west-1.amazonaws.com" if the bucket is located in Ireland). * Evidently, the bucket has to be hosted in the region denoted by the * endpoint for the test to succeed. *

* More info and the list of endpoint identifiers: - * http://docs.aws.amazon.com/general/latest/gr/rande.html#s3_region + * @see endpoint list. * * @throws Exception */ @Test public void testEndpoint() throws Exception { conf = new Configuration(); - String endpoint = conf.getTrimmed(TEST_ENDPOINT, ""); + String endpoint = conf.getTrimmed( + S3ATestConstants.CONFIGURATION_TEST_ENDPOINT, ""); if (endpoint.isEmpty()) { - LOG.warn("Custom endpoint test skipped as " + TEST_ENDPOINT + "config " + + LOG.warn("Custom endpoint test skipped as " + + S3ATestConstants.CONFIGURATION_TEST_ENDPOINT + "config " + "setting was not detected"); } else { conf.set(Constants.ENDPOINT, endpoint); diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileOperationCost.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileOperationCost.java index 8616e263ce0374..7fb54b1ac7e5af 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileOperationCost.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileOperationCost.java @@ -232,7 +232,8 @@ public void testFakeDirectoryDeletion() throws Throwable { int destDirDepth = directoriesInPath(destDir); directoriesCreated.assertDiffEquals(state, 1); -/* TODO: uncomment once HADOOP-13222 is in +/* TODO: uncomment once HADOOP-13222 "s3a.mkdirs() to delete empty fake parent directories" + is in deleteRequests.assertDiffEquals(state,1); directoriesDeleted.assertDiffEquals(state,0); fakeDirectoriesDeleted.assertDiffEquals(state,destDirDepth); diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestConstants.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestConstants.java index daaf948f9ada36..8c22f478d5c22e 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestConstants.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestConstants.java @@ -134,18 +134,11 @@ public interface S3ATestConstants { */ int DEFAULT_DIRECTORY_COUNT = 2; - /** - * Default scale test timeout in seconds: {@value}. - */ - int DEFAULT_TEST_TIMEOUT = 30 * 60; - /** * Default policy on scale tests: {@value}. */ boolean DEFAULT_SCALE_TESTS_ENABLED = false; - String KEY_ENCRYPTION_TESTS = TEST_FS_S3A + "encryption.enabled"; - /** * Fork ID passed down from maven if the test is running in parallel. */ @@ -159,7 +152,15 @@ public interface S3ATestConstants { int S3A_TEST_TIMEOUT = 10 * 60 * 1000; /** - * Timeout in Milliseconds for Scale Tests: {@value}. + * Timeout in Seconds for Scale Tests: {@value}. + */ + int SCALE_TEST_TIMEOUT_SECONDS = 30 * 60; + + int SCALE_TEST_TIMEOUT_MILLIS = SCALE_TEST_TIMEOUT_SECONDS * 1000; + /** + * Optional custom endpoint for S3A configuration tests. + * This does not set the endpoint for s3 access elsewhere. */ - int SCALE_TEST_TIMEOUT = 30 * 60 * 1000; + String CONFIGURATION_TEST_ENDPOINT = + "test.fs.s3a.endpoint"; } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java index 5d52b3722ef2a1..a09dcc2d19c3e2 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java @@ -59,7 +59,7 @@ public class S3ATestUtils { */ public static S3AFileSystem createTestFileSystem(Configuration conf) throws IOException { - return createTestFileSystem(conf, true); + return createTestFileSystem(conf, false); } /** diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/AbstractSTestS3AHugeFiles.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/AbstractSTestS3AHugeFiles.java index a60d0843a4511e..fcb6444349cccb 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/AbstractSTestS3AHugeFiles.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/AbstractSTestS3AHugeFiles.java @@ -38,6 +38,7 @@ import org.apache.hadoop.fs.StorageStatistics; import org.apache.hadoop.fs.contract.ContractTestUtils; import org.apache.hadoop.fs.s3a.S3AFileStatus; +import org.apache.hadoop.fs.s3a.S3AFileSystem; import org.apache.hadoop.fs.s3a.Statistic; import org.apache.hadoop.util.Progressable; @@ -70,27 +71,22 @@ public abstract class AbstractSTestS3AHugeFiles extends S3AScaleTestBase { private int partitionSize; @Override - public void setUp() throws Exception { - super.setUp(); - + public void setup() throws Exception { + super.setup(); final Path testPath = getTestPath(); scaleTestDir = new Path(testPath, "scale"); hugefile = new Path(scaleTestDir, "hugefile"); hugefileRenamed = new Path(scaleTestDir, "hugefileRenamed"); } - @Override - public void tearDown() throws Exception { - // do nothing. Specifically: do not delete the test dir - } /** * Note that this can get called before test setup. * @return the configuration to use. */ @Override - protected Configuration createConfiguration() { - Configuration conf = super.createConfiguration(); + protected Configuration createScaleConfiguration() { + Configuration conf = super.createScaleConfiguration(); partitionSize = (int)getTestPropertyBytes(conf, KEY_HUGE_PARTITION_SIZE, DEFAULT_PARTITION_SIZE); @@ -155,6 +151,7 @@ public void test_010_CreateHugeFile() throws IOException { // perform the upload. // there's lots of logging here, so that a tail -f on the output log // can give a view of what is happening. + S3AFileSystem fs = getFileSystem(); StorageStatistics storageStatistics = fs.getStorageStatistics(); String putRequests = Statistic.OBJECT_PUT_REQUESTS.getSymbol(); String putBytes = Statistic.OBJECT_PUT_BYTES.getSymbol(); @@ -286,12 +283,13 @@ private void verifyNoFailures(String operation) { } void assumeHugeFileExists() throws IOException { + S3AFileSystem fs = getFileSystem(); ContractTestUtils.assertPathExists(fs, "huge file not created", hugefile); ContractTestUtils.assertIsFile(fs, hugefile); } private void logFSState() { - LOG.info("File System state after operation:\n{}", fs); + LOG.info("File System state after operation:\n{}", getFileSystem()); } @Test @@ -305,6 +303,7 @@ public void test_040_PositionedReadHugeFile() throws Throwable { } String filetype = encrypted ? "encrypted file" : "file"; describe("Positioned reads of %s %s", filetype, hugefile); + S3AFileSystem fs = getFileSystem(); S3AFileStatus status = fs.getFileStatus(hugefile); long filesize = status.getLen(); int ops = 0; @@ -344,6 +343,7 @@ public void test_040_PositionedReadHugeFile() throws Throwable { public void test_050_readHugeFile() throws Throwable { assumeHugeFileExists(); describe("Reading %s", hugefile); + S3AFileSystem fs = getFileSystem(); S3AFileStatus status = fs.getFileStatus(hugefile); long filesize = status.getLen(); long blocks = filesize / uploadBlockSize; @@ -369,6 +369,7 @@ public void test_050_readHugeFile() throws Throwable { public void test_100_renameHugeFile() throws Throwable { assumeHugeFileExists(); describe("renaming %s to %s", hugefile, hugefileRenamed); + S3AFileSystem fs = getFileSystem(); S3AFileStatus status = fs.getFileStatus(hugefile); long filesize = status.getLen(); fs.delete(hugefileRenamed, false); @@ -396,7 +397,7 @@ public void test_100_renameHugeFile() throws Throwable { public void test_999_DeleteHugeFiles() throws IOException { deleteHugeFile(); ContractTestUtils.NanoTimer timer2 = new ContractTestUtils.NanoTimer(); - + S3AFileSystem fs = getFileSystem(); fs.delete(hugefileRenamed, false); timer2.end("time to delete %s", hugefileRenamed); ContractTestUtils.rm(fs, getTestPath(), true, true); @@ -405,7 +406,7 @@ public void test_999_DeleteHugeFiles() throws IOException { protected void deleteHugeFile() throws IOException { describe("Deleting %s", hugefile); NanoTimer timer = new NanoTimer(); - fs.delete(hugefile, false); + getFileSystem().delete(hugefile, false); timer.end("time to delete %s", hugefile); } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3ADeleteFilesOneByOne.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3ADeleteFilesOneByOne.java index b054f163005116..10dfa65a150982 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3ADeleteFilesOneByOne.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3ADeleteFilesOneByOne.java @@ -27,8 +27,8 @@ public class ITestS3ADeleteFilesOneByOne extends ITestS3ADeleteManyFiles { @Override - protected Configuration createConfiguration() { - Configuration configuration = super.createConfiguration(); + protected Configuration createScaleConfiguration() { + Configuration configuration = super.createScaleConfiguration(); configuration.setBoolean(Constants.ENABLE_MULTI_DELETE, false); return configuration; } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3ADeleteManyFiles.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3ADeleteManyFiles.java index fbbc58f8aa68e3..d4b6dd97d55fc7 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3ADeleteManyFiles.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3ADeleteManyFiles.java @@ -20,6 +20,8 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.contract.ContractTestUtils; +import org.apache.hadoop.fs.s3a.S3AFileSystem; + import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -54,8 +56,8 @@ public void testBulkRenameAndDelete() throws Throwable { final Path srcDir = new Path(scaleTestDir, "src"); final Path finalDir = new Path(scaleTestDir, "final"); final long count = getOperationCount(); + final S3AFileSystem fs = getFileSystem(); ContractTestUtils.rm(fs, scaleTestDir, true, false); - fs.mkdirs(srcDir); fs.mkdirs(finalDir); diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3ADirectoryPerformance.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3ADirectoryPerformance.java index 1d0f9e1c3a7d6b..d71364f4fcd2f7 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3ADirectoryPerformance.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3ADirectoryPerformance.java @@ -19,6 +19,7 @@ package org.apache.hadoop.fs.s3a.scale; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.s3a.S3AFileSystem; import org.apache.hadoop.fs.s3a.Statistic; import org.junit.Test; import org.slf4j.Logger; @@ -42,6 +43,7 @@ public void testListOperations() throws Throwable { describe("Test recursive list operations"); final Path scaleTestDir = path("testListOperations"); final Path listDir = new Path(scaleTestDir, "lists"); + S3AFileSystem fs = getFileSystem(); // scale factor. int scale = getConf().getInt(KEY_DIRECTORY_COUNT, DEFAULT_DIRECTORY_COUNT); @@ -138,7 +140,7 @@ public void testListOperations() throws Throwable { public void testTimeToStatEmptyDirectory() throws Throwable { describe("Time to stat an empty directory"); Path path = path("empty"); - fs.mkdirs(path); + getFileSystem().mkdirs(path); timeToStatPath(path); } @@ -146,6 +148,7 @@ public void testTimeToStatEmptyDirectory() throws Throwable { public void testTimeToStatNonEmptyDirectory() throws Throwable { describe("Time to stat a non-empty directory"); Path path = path("dir"); + S3AFileSystem fs = getFileSystem(); fs.mkdirs(path); touch(fs, new Path(path, "file")); timeToStatPath(path); @@ -155,7 +158,7 @@ public void testTimeToStatNonEmptyDirectory() throws Throwable { public void testTimeToStatFile() throws Throwable { describe("Time to stat a simple file"); Path path = path("file"); - touch(fs, path); + touch(getFileSystem(), path); timeToStatPath(path); } @@ -167,6 +170,7 @@ public void testTimeToStatRoot() throws Throwable { private void timeToStatPath(Path path) throws IOException { describe("Timing getFileStatus(\"%s\")", path); + S3AFileSystem fs = getFileSystem(); MetricDiff metadataRequests = new MetricDiff(fs, Statistic.OBJECT_METADATA_REQUESTS); MetricDiff listRequests = diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3AHugeFilesClassicOutput.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3AHugeFilesClassicOutput.java index 45eef2491a8842..551956bd8d27db 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3AHugeFilesClassicOutput.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3AHugeFilesClassicOutput.java @@ -29,8 +29,8 @@ public class ITestS3AHugeFilesClassicOutput extends AbstractSTestS3AHugeFiles { @Override - protected Configuration createConfiguration() { - final Configuration conf = super.createConfiguration(); + protected Configuration createScaleConfiguration() { + final Configuration conf = super.createScaleConfiguration(); conf.setBoolean(Constants.FAST_UPLOAD, false); return conf; } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3AInputStreamPerformance.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3AInputStreamPerformance.java index 090acc6c3482dd..cc8187e6276ea8 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3AInputStreamPerformance.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3AInputStreamPerformance.java @@ -436,6 +436,7 @@ public void testRandomReadOverBuffer() throws Throwable { describe("read over a buffer, making sure that the requests" + " spans readahead ranges"); int datasetLen = _32K; + S3AFileSystem fs = getFileSystem(); Path dataFile = path("testReadOverBuffer.bin"); byte[] sourceData = dataset(datasetLen, 0, 64); // relies on the field 'fs' referring to the R/W FS diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/S3AScaleTestBase.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/S3AScaleTestBase.java index 7a929f3d2a0cd7..c4174bfd63be1f 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/S3AScaleTestBase.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/S3AScaleTestBase.java @@ -21,22 +21,15 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.contract.ContractTestUtils; import org.apache.hadoop.fs.s3a.AbstractS3ATestBase; -import org.apache.hadoop.fs.s3a.S3AFileSystem; import org.apache.hadoop.fs.s3a.S3AInputStream; import org.apache.hadoop.fs.s3a.S3AInstrumentation; import org.apache.hadoop.fs.s3a.S3ATestConstants; import org.apache.hadoop.fs.s3a.Statistic; import org.apache.hadoop.metrics2.lib.MutableGaugeLong; -import org.junit.After; + import org.junit.Assert; import org.junit.Assume; -import org.junit.Before; -import org.junit.Rule; -import org.junit.rules.TestName; -import org.junit.rules.Timeout; -import org.junit.Assert; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -47,17 +40,35 @@ /** * Base class for scale tests; here is where the common scale configuration * keys are defined. + *

+ * Configuration setup is a bit more complex than in the parent classes, + * as the test timeout is desired prior to the {@link #getTestTimeoutMillis()} + * being called to set the test timeout rule; this happens before any of + * the methods tagged with {@code @Before} are invoked. + *

+ * The algorithm is: + *

    + *
  1. Create a configuration on demand, via + * {@link #demandCreateConfiguration()}
    2. + *
  2. Have that return the value of {@link #conf} or create a new one + * if that field is null (and set the field to the created value).
    3. + *
  3. Override the superclasses {@link #createConfiguration()} + * to return the demand created value; make that method final so that + * subclasses don't break things by overridding it.
    4. + *
  4. Add a new override point {@link #createScaleConfiguration()} + * to create the config, one which subclasses can (and do) override.
    5. + *
+ * Bear in mind that this process also takes place during initialization + * of the superclass; the overridden methods are being invoked before + * their instances are fully configured. This is considered + * very bad form in Java code (indeed, in C++ it is actually permitted; + * the base class implementations get invoked instead). */ public class S3AScaleTestBase extends AbstractS3ATestBase { - @Rule - public Timeout testTimeout = createTestTimeout(); - public static final int _1KB = 1024; public static final int _1MB = _1KB * _1KB; - protected S3AFileSystem fs; - protected static final Logger LOG = LoggerFactory.getLogger(S3AScaleTestBase.class); @@ -65,14 +76,7 @@ public class S3AScaleTestBase extends AbstractS3ATestBase { private boolean enabled; - /** - * Configuration generator. May be overridden to inject - * some custom options. - * @return a configuration with which to create FS instances - */ - protected Configuration createConfiguration() { - return new Configuration(); - } + private Path testPath; /** @@ -83,40 +87,49 @@ public Configuration getConf() { return conf; } - /** - * Setup. This triggers creation of the configuration. - */ - @Before - public void setUp() throws Exception { - demandCreateConfiguration(); + @Override + public void setup() throws Exception { + super.setup(); + testPath = path("/tests3ascale"); LOG.debug("Scale test operation count = {}", getOperationCount()); // multipart purges are disabled on the scale tests - fs = createTestFileSystem(conf, false); // check for the test being enabled enabled = getTestPropertyBool( getConf(), KEY_SCALE_TESTS_ENABLED, DEFAULT_SCALE_TESTS_ENABLED); Assume.assumeTrue("Scale test disabled: to enable set property " + - KEY_SCALE_TESTS_ENABLED, enabled); + KEY_SCALE_TESTS_ENABLED, isEnabled()); } /** - * Create the configuration if it is not already set up. + * Create the configuration if it is not already set up, calling + * {@link #createScaleConfiguration()} to do so. * @return the configuration. */ private synchronized Configuration demandCreateConfiguration() { if (conf == null) { - conf = createConfiguration(); + conf = createScaleConfiguration(); } - return getConfiguration(); + return conf; } - @Override - public void setup() throws Exception { - super.setup(); - testPath = path("/tests3a"); - fs = getFileSystem(); + /** + * Returns the config created with {@link #demandCreateConfiguration()}. + * Subclasses must override {@link #createScaleConfiguration()} + * in order to customize their configurations. + * @return a configuration with which to create FS instances + */ + protected final Configuration createConfiguration() { + return demandCreateConfiguration(); + } + + /** + * Override point: create a configuration. + * @return a configuration with which to create FS instances + */ + protected Configuration createScaleConfiguration() { + return new Configuration(); } protected Path getTestPath() { @@ -127,35 +140,19 @@ protected long getOperationCount() { return getConf().getLong(KEY_OPERATION_COUNT, DEFAULT_OPERATION_COUNT); } - /** - * Create the timeout for tests. Some large tests may need a larger value. - * @return the test timeout to use - */ - protected Timeout createTestTimeout() { - demandCreateConfiguration(); - return new Timeout( - getTestTimeoutSeconds() * 1000); - } - /** * Get the test timeout in seconds. * @return the test timeout as set in system properties or the default. */ - protected static int getTestTimeoutSeconds() { - return getTestPropertyInt(null, + protected int getTestTimeoutSeconds() { + return getTestPropertyInt(demandCreateConfiguration(), KEY_TEST_TIMEOUT, - DEFAULT_TEST_TIMEOUT); + SCALE_TEST_TIMEOUT_SECONDS); } - /** - * Describe a test in the logs. - * @param text text to print - * @param args arguments to format in the printing - */ - protected void describe(String text, Object... args) { - LOG.info("\n\n{}: {}\n", - methodName.getMethodName(), - String.format(text, args)); + @Override + protected int getTestTimeoutMillis() { + return getTestTimeoutSeconds() * 1000; } /** @@ -184,20 +181,25 @@ protected S3AInstrumentation.InputStreamStatistics getInputStreamStatistics( * @return the value. */ public long gaugeValue(Statistic statistic) { - S3AInstrumentation instrumentation = fs.getInstrumentation(); + S3AInstrumentation instrumentation = getFileSystem().getInstrumentation(); MutableGaugeLong gauge = instrumentation.lookupGauge(statistic.getSymbol()); assertNotNull("No gauge " + statistic + " in " + instrumentation.dump("", " = ", "\n", true), gauge); return gauge.value(); } - protected boolean isEnabled() { + /** + * Is the test enabled; this is controlled by the configuration + * and the {@code -Dscale} maven option. + * @return true if the scale tests are enabled. + */ + protected final boolean isEnabled() { return enabled; } /** - * Flag to indicate that this test is being used sequentially. This - * is used by some of the scale tests to validate test time expectations. + * Flag to indicate that this test is being executed in parallel. + * This is used by some of the scale tests to validate test time expectations. * @return true if the build indicates this test is being run in parallel. */ protected boolean isParallelExecution() { diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/yarn/ITestS3AMiniYarnCluster.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/yarn/ITestS3AMiniYarnCluster.java index ac8caf8e79ab8c..5b42d4d1981c14 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/yarn/ITestS3AMiniYarnCluster.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/yarn/ITestS3AMiniYarnCluster.java @@ -28,6 +28,7 @@ import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileContext; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.contract.ContractTestUtils; import org.apache.hadoop.fs.s3a.AbstractS3ATestBase; import org.apache.hadoop.fs.s3a.S3AFileSystem; import org.apache.hadoop.fs.s3a.S3ATestUtils; @@ -71,8 +72,9 @@ public void setup() throws Exception { @Override public void teardown() throws Exception { - fs.delete(rootPath, true); - yarnCluster.stop(); + if (yarnCluster != null) { + yarnCluster.stop(); + } super.teardown(); } @@ -136,15 +138,9 @@ private void writeStringToFile(Path path, String string) throws IOException { /** * helper method. */ - private String readStringFromFile(Path path) { - try (FSDataInputStream in = fs.open(path)) { - long bytesLen = fs.getFileStatus(path).getLen(); - byte[] buffer = new byte[(int) bytesLen]; - IOUtils.readFully(in, buffer, 0, buffer.length); - return new String(buffer); - } catch (IOException e) { - throw new RuntimeException("Failed to read from [" + path + "]", e); - } + private String readStringFromFile(Path path) throws IOException { + return ContractTestUtils.readBytesToString(fs, path, + (int) fs.getFileStatus(path).getLen()); } } From f94d2678f7ce9137e813cd5dec239080100a8f11 Mon Sep 17 00:00:00 2001 From: Steve Loughran Date: Mon, 24 Oct 2016 18:05:11 +0100 Subject: [PATCH 7/7] HADOOP-13614: clean up imports --- .../org/apache/hadoop/fs/s3a/ITestS3AFailureHandling.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFailureHandling.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFailureHandling.java index 8116c2c80863d7..7cd109438253d3 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFailureHandling.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFailureHandling.java @@ -21,9 +21,6 @@ import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.contract.AbstractFSContract; -import org.apache.hadoop.fs.contract.AbstractFSContractTestBase; -import org.apache.hadoop.fs.contract.s3a.S3AContract; import org.apache.hadoop.test.LambdaTestUtils; import org.junit.Test; @@ -35,7 +32,6 @@ import java.util.concurrent.Callable; import static org.apache.hadoop.fs.contract.ContractTestUtils.*; -import static org.apache.hadoop.fs.s3a.S3ATestUtils.*; /** * Test S3A Failure translation, including a functional test