From 1925dd1ede28d02384e0fd8847605f5f28daef3d Mon Sep 17 00:00:00 2001 From: Anuj Modi Date: Sat, 1 Feb 2025 03:00:34 -0800 Subject: [PATCH 01/13] Adding Tests for Blob Endpoint Related Scenarios --- .../azurebfs/AbstractAbfsIntegrationTest.java | 30 ++++ .../ITestAbfsListStatusRemoteIterator.java | 1 + .../ITestAzureBlobFileSystemAttributes.java | 164 ++++++++++++------ .../ITestAzureBlobFileSystemChooseSAS.java | 3 +- .../ITestAzureBlobFileSystemFileStatus.java | 115 +++++++++++- .../contract/TestBlobListXmlParser.java | 154 ++++++++++++++++ ...maTest.java => TestDfsListJsonParser.java} | 2 +- .../services/ITestAbfsClientHandler.java | 1 + 8 files changed, 411 insertions(+), 59 deletions(-) create mode 100644 hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/TestBlobListXmlParser.java rename hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/{ListResultSchemaTest.java => TestDfsListJsonParser.java} (99%) diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java index c7da1c7b23160f..6d3bbad761c8fb 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java @@ -25,6 +25,7 @@ import java.util.UUID; import java.util.concurrent.Callable; +import org.assertj.core.api.Assertions; import org.junit.After; import org.junit.Assume; import org.junit.Before; @@ -64,6 +65,7 @@ import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.FORWARD_SLASH; import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.*; import static org.apache.hadoop.fs.azurebfs.constants.FileSystemUriSchemes.ABFS_BLOB_DOMAIN_NAME; +import static org.apache.hadoop.fs.azurebfs.constants.FileSystemUriSchemes.ABFS_DFS_DOMAIN_NAME; import static org.apache.hadoop.fs.azurebfs.constants.FileSystemUriSchemes.HTTPS_SCHEME; import static org.apache.hadoop.fs.azurebfs.contracts.services.AzureServiceErrorCode.FILE_SYSTEM_NOT_FOUND; import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.*; @@ -636,4 +638,32 @@ protected void assumeBlobServiceType() { Assume.assumeTrue("Blob service type is required for this test", getAbfsServiceType() == AbfsServiceType.BLOB); } + + protected void assumeDfsServiceType() { + Assume.assumeTrue("DFS service type is required for this test", + getAbfsServiceType() == AbfsServiceType.DFS); + } + + protected void assumeHnsEnabled() throws IOException { + Assume.assumeTrue("HNS-Enabled account must be used for this test", + getIsNamespaceEnabled(getFileSystem())); + } + + protected void assumeHnsDisabled() throws IOException { + Assume.assumeFalse("HNS-Enabled account must not be used for this test", + getIsNamespaceEnabled(getFileSystem())); + } + + /** + * Assert that the path contains the expected DNS suffix. + * If service type is blob, then path should have blob domain name. + * @param path to be asserted. + */ + protected void assertPathDns(Path path) { + String expectedDns = getAbfsServiceType() == AbfsServiceType.BLOB + ? ABFS_BLOB_DOMAIN_NAME : ABFS_DFS_DOMAIN_NAME; + Assertions.assertThat(path.toString()) + .describedAs("Path does not contain expected DNS") + .contains(expectedDns); + } } diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsListStatusRemoteIterator.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsListStatusRemoteIterator.java index ea1d0e26facec6..6f7fe0ce2cab24 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsListStatusRemoteIterator.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsListStatusRemoteIterator.java @@ -225,6 +225,7 @@ public void testNonExistingPath() throws Exception { private void verifyIteratorResultContent(FileStatus fileStatus, List fileNames) { + assertPathDns(fileStatus.getPath()); String pathStr = fileStatus.getPath().toString(); Assert.assertTrue( String.format("Could not remove path %s from filenames %s", pathStr, diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemAttributes.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemAttributes.java index a4ad0d207c3fd3..f6d1dc2674b4b4 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemAttributes.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemAttributes.java @@ -18,6 +18,7 @@ package org.apache.hadoop.fs.azurebfs; +import java.io.FileNotFoundException; import java.io.IOException; import java.util.EnumSet; @@ -28,6 +29,7 @@ import org.apache.hadoop.fs.XAttrSetFlag; import org.apache.hadoop.fs.azurebfs.constants.FSOperationType; import org.apache.hadoop.fs.azurebfs.contracts.exceptions.AbfsRestOperationException; +import org.apache.hadoop.fs.azurebfs.utils.DirectoryStateHelper; import org.apache.hadoop.fs.azurebfs.utils.TracingHeaderValidator; import static java.net.HttpURLConnection.HTTP_BAD_REQUEST; @@ -56,16 +58,15 @@ public void testSetGetXAttr() throws Exception { @Test public void testSetGetXAttrCreateReplace() throws Exception { AzureBlobFileSystem fs = getFileSystem(); - byte[] attributeValue = fs.getAbfsStore().encodeAttribute("one"); + String decodedAttributeValue = "one"; + byte[] attributeValue = fs.getAbfsStore().encodeAttribute(decodedAttributeValue); String attributeName = "user.someAttribute"; Path testFile = path(getMethodName()); // after creating a file, it must be possible to create a new xAttr touch(testFile); fs.setXAttr(testFile, attributeName, attributeValue, CREATE_FLAG); - Assertions.assertThat(fs.getXAttr(testFile, attributeName)) - .describedAs("Retrieved Attribute Value is Not as Expected") - .containsExactly(attributeValue); + assertAttributeEqual(fs.getXAttr(testFile, attributeName), attributeValue, decodedAttributeValue); // however after the xAttr is created, creating it again must fail intercept(IOException.class, () -> fs.setXAttr(testFile, attributeName, attributeValue, CREATE_FLAG)); @@ -74,8 +75,10 @@ public void testSetGetXAttrCreateReplace() throws Exception { @Test public void testSetGetXAttrReplace() throws Exception { AzureBlobFileSystem fs = getFileSystem(); - byte[] attributeValue1 = fs.getAbfsStore().encodeAttribute("one"); - byte[] attributeValue2 = fs.getAbfsStore().encodeAttribute("two"); + String decodedAttribute1 = "one"; + String decodedAttribute2 = "two"; + byte[] attributeValue1 = fs.getAbfsStore().encodeAttribute(decodedAttribute1); + byte[] attributeValue2 = fs.getAbfsStore().encodeAttribute(decodedAttribute2); String attributeName = "user.someAttribute"; Path testFile = path(getMethodName()); @@ -88,9 +91,89 @@ public void testSetGetXAttrReplace() throws Exception { // however after the xAttr is created, replacing it must succeed fs.setXAttr(testFile, attributeName, attributeValue1, CREATE_FLAG); fs.setXAttr(testFile, attributeName, attributeValue2, REPLACE_FLAG); - Assertions.assertThat(fs.getXAttr(testFile, attributeName)) - .describedAs("Retrieved Attribute Value is Not as Expected") - .containsExactly(attributeValue2); + assertAttributeEqual(fs.getXAttr(testFile, attributeName), attributeValue2, + decodedAttribute2); + } + + @Test + public void testGetSetXAttrOnRoot() throws Exception { + AzureBlobFileSystem fs = getFileSystem(); + String attributeName = "user.attribute1"; + byte[] attributeValue = fs.getAbfsStore().encodeAttribute("hi"); + final Path testPath = new Path(ROOT_PATH); + + AbfsRestOperationException ex = intercept(AbfsRestOperationException.class, () -> { + fs.getXAttr(testPath, attributeName); + }); + + Assertions.assertThat(ex.getStatusCode()) + .describedAs("GetXAttr() on root should fail with Bad Request") + .isEqualTo(HTTP_BAD_REQUEST); + + ex = intercept(AbfsRestOperationException.class, () -> { + fs.setXAttr(testPath, attributeName, attributeValue, CREATE_FLAG); + }); + + Assertions.assertThat(ex.getStatusCode()) + .describedAs("SetXAttr() on root should fail with Bad Request") + .isEqualTo(HTTP_BAD_REQUEST); + } + @Test + public void testGetSetXAttrOnExplicitDir() throws Exception { + AzureBlobFileSystem fs = getFileSystem(); + final Path testPath = path(getMethodName()); + fs.mkdirs(testPath); + testGetSetXAttrHelper(fs, testPath); + + // Assert that the folder is now explicit + DirectoryStateHelper.isExplicitDirectory(testPath, fs, getTestTracingContext(fs, true)); + } + @Test + public void testGetSetXAttrOnNonExistingPath() throws Exception { + AzureBlobFileSystem fs = getFileSystem(); + final Path testPath = path(getMethodName()); + intercept( + FileNotFoundException.class, () -> testGetSetXAttrHelper(fs, testPath)); + } + + /** + * Trying to set same attribute multiple times should result in no failure + * @throws Exception if test fails + */ + @Test + public void testSetXAttrMultipleOperations() throws Exception { + AzureBlobFileSystem fs = getFileSystem(); + final Path path = path(getMethodName()); + fs.create(path); + + String attributeName = "user.attribute1"; + String decodedAttributeValue = "hi"; + byte[] attributeValue = fs.getAbfsStore().encodeAttribute(decodedAttributeValue); + + // Attribute not present initially + assertAttributeNull(fs.getXAttr(path, attributeName)); + + // Set the Attributes Multiple times + // Filesystem internally adds create and replace flags + fs.setXAttr(path, attributeName, attributeValue); + fs.setXAttr(path, attributeName, attributeValue); + + // Check if the attribute is retrievable + byte[] rv = fs.getXAttr(path, attributeName); + assertAttributeEqual(rv, attributeValue, decodedAttributeValue); + } + + @Test + public void testGetSetXAttrOnImplicitDir() throws Exception { + assumeBlobServiceType(); + AzureBlobFileSystem fs = getFileSystem(); + final Path testPath = path(getMethodName()); + createAzCopyFolder(testPath); + testGetSetXAttrHelper(fs, testPath); + + // Assert that the folder is now explicit + DirectoryStateHelper.isExplicitDirectory(testPath, fs, getTestTracingContext(fs, true)); + DirectoryStateHelper.isExplicitDirectory(testPath.getParent(), fs, getTestTracingContext(fs, true)); } private void testGetSetXAttrHelper(final AzureBlobFileSystem fs, @@ -104,12 +187,8 @@ private void testGetSetXAttrHelper(final AzureBlobFileSystem fs, byte[] attributeValue2 = fs.getAbfsStore().encodeAttribute(decodedAttributeValue2); // Attribute not present initially - Assertions.assertThat(fs.getXAttr(testPath, attributeName1)) - .describedAs("Cannot get attribute before setting it") - .isNull(); - Assertions.assertThat(fs.getXAttr(testPath, attributeName2)) - .describedAs("Cannot get attribute before setting it") - .isNull(); + assertAttributeNull(fs.getXAttr(testPath, attributeName1)); + assertAttributeNull(fs.getXAttr(testPath, attributeName2)); // Set the Attributes fs.registerListener( @@ -121,12 +200,7 @@ private void testGetSetXAttrHelper(final AzureBlobFileSystem fs, // Check if the attribute is retrievable fs.setListenerOperation(FSOperationType.GET_ATTR); byte[] rv = fs.getXAttr(testPath, attributeName1); - Assertions.assertThat(rv) - .describedAs("Retrieved Attribute Does not Matches in Encoded Form") - .containsExactly(attributeValue1); - Assertions.assertThat(fs.getAbfsStore().decodeAttribute(rv)) - .describedAs("Retrieved Attribute Does not Matches in Decoded Form") - .isEqualTo(decodedAttributeValue1); + assertAttributeEqual(rv, attributeValue1, decodedAttributeValue1); fs.registerListener(null); // Set the second Attribute @@ -134,43 +208,25 @@ private void testGetSetXAttrHelper(final AzureBlobFileSystem fs, // Check all the attributes present and previous Attribute not overridden rv = fs.getXAttr(testPath, attributeName1); - Assertions.assertThat(rv) - .describedAs("Retrieved Attribute Does not Matches in Encoded Form") - .containsExactly(attributeValue1); - Assertions.assertThat(fs.getAbfsStore().decodeAttribute(rv)) - .describedAs("Retrieved Attribute Does not Matches in Decoded Form") - .isEqualTo(decodedAttributeValue1); + assertAttributeEqual(rv, attributeValue1, decodedAttributeValue1); rv = fs.getXAttr(testPath, attributeName2); - Assertions.assertThat(rv) - .describedAs("Retrieved Attribute Does not Matches in Encoded Form") - .containsExactly(attributeValue2); - Assertions.assertThat(fs.getAbfsStore().decodeAttribute(rv)) - .describedAs("Retrieved Attribute Does not Matches in Decoded Form") - .isEqualTo(decodedAttributeValue2); + assertAttributeEqual(rv, attributeValue2, decodedAttributeValue2); } - @Test - public void testGetSetXAttrOnRoot() throws Exception { - AzureBlobFileSystem fs = getFileSystem(); - String attributeName = "user.attribute1"; - byte[] attributeValue = fs.getAbfsStore().encodeAttribute("hi"); - final Path testPath = new Path(ROOT_PATH); - - AbfsRestOperationException ex = intercept(AbfsRestOperationException.class, () -> { - fs.getXAttr(testPath, attributeName); - }); - - Assertions.assertThat(ex.getStatusCode()) - .describedAs("GetXAttr() on root should fail with Bad Request") - .isEqualTo(HTTP_BAD_REQUEST); - - ex = intercept(AbfsRestOperationException.class, () -> { - fs.setXAttr(testPath, attributeName, attributeValue, CREATE_FLAG); - }); + private void assertAttributeNull(byte[] rv) { + Assertions.assertThat(rv) + .describedAs("Cannot get attribute before setting it") + .isNull(); + } - Assertions.assertThat(ex.getStatusCode()) - .describedAs("SetXAttr() on root should fail with Bad Request") - .isEqualTo(HTTP_BAD_REQUEST); + private void assertAttributeEqual(byte[] rv, byte[] attributeValue, + String decodedAttributeValue) throws Exception { + Assertions.assertThat(rv) + .describedAs("Retrieved Attribute Does not Matches in Encoded Form") + .containsExactly(attributeValue); + Assertions.assertThat(getFileSystem().getAbfsStore().decodeAttribute(rv)) + .describedAs("Retrieved Attribute Does not Matches in Decoded Form") + .isEqualTo(decodedAttributeValue); } } diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemChooseSAS.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemChooseSAS.java index 29170e187e27c9..a0e6a3b5192767 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemChooseSAS.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemChooseSAS.java @@ -93,7 +93,8 @@ private void generateAccountSAS() throws AzureBlobFileSystemException { */ @Test public void testBothProviderFixedTokenConfigured() throws Exception { - Assume.assumeTrue(getAbfsServiceType() == AbfsServiceType.DFS && getIsNamespaceEnabled(getFileSystem())); + assumeDfsServiceType(); + assumeHnsEnabled(); AbfsConfiguration testAbfsConfig = new AbfsConfiguration( getRawConfiguration(), this.getAccountName()); removeAnyPresetConfiguration(testAbfsConfig); diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemFileStatus.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemFileStatus.java index 7bd645ecd0b235..9bb7d1cc7aef61 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemFileStatus.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemFileStatus.java @@ -18,16 +18,24 @@ package org.apache.hadoop.fs.azurebfs; + +import java.io.FileNotFoundException; import java.io.IOException; -import org.apache.hadoop.fs.CommonConfigurationKeys; +import org.assertj.core.api.Assertions; import org.junit.Test; +import org.mockito.Mockito; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.azurebfs.services.AbfsBlobClient; import org.apache.hadoop.fs.permission.FsPermission; +import static org.apache.hadoop.fs.CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY; import static org.apache.hadoop.fs.contract.ContractTestUtils.assertPathExists; +import static org.apache.hadoop.test.LambdaTestUtils.intercept; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; /** * Test FileStatus. @@ -58,7 +66,7 @@ public void testEnsureStatusWorksForRoot() throws Exception { @Test public void testFileStatusPermissionsAndOwnerAndGroup() throws Exception { final AzureBlobFileSystem fs = this.getFileSystem(); - fs.getConf().set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, DEFAULT_UMASK_VALUE); + fs.getConf().set(FS_PERMISSIONS_UMASK_KEY, DEFAULT_UMASK_VALUE); Path testFile = path(TEST_FILE); touch(testFile); validateStatus(fs, testFile, false); @@ -97,7 +105,7 @@ private FileStatus validateStatus(final AzureBlobFileSystem fs, final Path name, @Test public void testFolderStatusPermissionsAndOwnerAndGroup() throws Exception { final AzureBlobFileSystem fs = this.getFileSystem(); - fs.getConf().set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, DEFAULT_UMASK_VALUE); + fs.getConf().set(FS_PERMISSIONS_UMASK_KEY, DEFAULT_UMASK_VALUE); Path testFolder = path(TEST_FOLDER); fs.mkdirs(testFolder); @@ -147,6 +155,11 @@ public void testLastModifiedTime() throws IOException { createEndTime > lastModifiedTime); } + /** + * Test to verify fs.listStatus() works as expected on root path + * across account types and endpoints configured. + * @throws IOException if test fails + */ @Test public void testFileStatusOnRoot() throws IOException { AzureBlobFileSystem fs = getFileSystem(); @@ -162,6 +175,102 @@ public void testFileStatusOnRoot() throws IOException { // Assert that passing absolute root path without "/" works testPathStr = testPathStr.substring(0, testPathStr.length() - 1); validateStatus(fs, new Path(testPathStr), true); + } + + /** + * Test to verify fs.getFileStatus() works as expected on explicit paths as expected. + * Explicit path can exist as a directory as well as a file. + * @throws IOException if test fails + */ + @Test + public void testFileStatusOnExplicitPath() throws Exception { + AzureBlobFileSystem fs = getFileSystem(); + Path explicitDirPath = path("explicitDir"); + Path filePath = new Path(explicitDirPath, "explicitFile"); + Path nonExistingPath = new Path(explicitDirPath, "nonExistingFile"); + + fs.mkdirs(explicitDirPath); + fs.create(filePath).close(); + + // Test File Status on explicit dir path. + FileStatus fileStatus = fs.getFileStatus(explicitDirPath); + verifyFileStatus(fileStatus, true); + + // Test File Status on file with explicit parent. + fileStatus = fs.getFileStatus(filePath); + verifyFileStatus(fileStatus, false); + + // Test File Status non-existing file with explicit parent. + FileNotFoundException ex = intercept(FileNotFoundException.class, () -> { + fs.getFileStatus(nonExistingPath); + }); + verifyFileNotFound(ex, nonExistingPath.getName()); + } + + /** + * Test to verify fs.getFileStatus() works as expected on implicit paths as expected. + * Implicit path can exist as a directory only in HNS-Disabled Accounts. + * @throws Exception + */ + @Test + public void testFileStatusOnImplicitPath() throws Exception { + AzureBlobFileSystem fs = getFileSystem(); + Path filePath = path("implicitDir/fileWithImplicitParent"); + Path implicitDir = filePath.getParent(); + Path nonExistingPath = new Path(implicitDir, "nonExistingFile"); + + createAzCopyFile(filePath); + + // Test File Status on implicit dir parent. + FileStatus fileStatus = fs.getFileStatus(implicitDir); + verifyFileStatus(fileStatus, true); + + // Test File Status on file with implicit parent. + fileStatus = fs.getFileStatus(filePath); + verifyFileStatus(fileStatus, false); + + // Test File Status on non-existing file with implicit parent. + FileNotFoundException ex = intercept(FileNotFoundException.class, () -> { + fs.getFileStatus(nonExistingPath); + }); + verifyFileNotFound(ex, nonExistingPath.getName()); + } + + /** + * Test to verify fs.getFileStatus() need to internally call listStatus on path. + * @throws Exception if test fails + */ + @Test + public void testListStatusIsCalledForImplicitPathOnBlobEndpoint() throws Exception { + assumeBlobServiceType(); + AzureBlobFileSystem fs = Mockito.spy(getFileSystem()); + AzureBlobFileSystemStore store = Mockito.spy(fs.getAbfsStore()); + Mockito.doReturn(store).when(fs).getAbfsStore(); + AbfsBlobClient abfsClient = Mockito.spy(store.getClientHandler().getBlobClient()); + Mockito.doReturn(abfsClient).when(store).getClient(); + + Path implicitPath = path("implicitDir"); + createAzCopyFolder(implicitPath); + + fs.getFileStatus(implicitPath); + + Mockito.verify(abfsClient, Mockito.times(1)).getPathStatus(any(), eq(false), any(), any()); + Mockito.verify(abfsClient, Mockito.times(1)).listPath(any(), eq(false), eq(1), any(), any(), eq(false)); + } + + private void verifyFileStatus(FileStatus fileStatus, boolean isDir) { + Assertions.assertThat(fileStatus).isNotNull(); + if (isDir) { + Assertions.assertThat(fileStatus.getLen()).isEqualTo(0); + Assertions.assertThat(fileStatus.isDirectory()).isTrue(); + } else { + Assertions.assertThat(fileStatus.isFile()).isTrue(); + } + assertPathDns(fileStatus.getPath()); + } + private void verifyFileNotFound(FileNotFoundException ex, String key) { + Assertions.assertThat(ex).isNotNull(); + Assertions.assertThat(ex.getMessage()).contains(key); } } diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/TestBlobListXmlParser.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/TestBlobListXmlParser.java new file mode 100644 index 00000000000000..4bfb5e5a14a2db --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/TestBlobListXmlParser.java @@ -0,0 +1,154 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.azurebfs.contract; + +import javax.xml.parsers.ParserConfigurationException; +import javax.xml.parsers.SAXParser; +import javax.xml.parsers.SAXParserFactory; +import java.io.ByteArrayInputStream; +import java.io.InputStream; +import org.assertj.core.api.Assertions; +import java.util.List; + +import org.junit.Test; +import org.xml.sax.SAXException; + +import org.apache.hadoop.fs.azurebfs.contracts.services.BlobListResultEntrySchema; +import org.apache.hadoop.fs.azurebfs.contracts.services.BlobListResultSchema; +import org.apache.hadoop.fs.azurebfs.contracts.services.BlobListXmlParser; + +public class TestBlobListXmlParser { + @Test + public void testXMLParser() throws Exception { + String xmlResponseWithDelimiter = "" + + "" + + "" + + "/" + + "" + + "" + + "bye/" + + "" + + "" + + "explicitDir" + + "" + + "Tue, 06 Jun 2023 08:34:28 GMT" + + "Tue, 06 Jun 2023 08:35:00 GMT" + + "0x8DB6668EAB50E67" + + "0" + + "application/octet-stream" + + "" + + "" + + "" + + "1B2M2Y8AsgTpgAmY7PhCfg==" + + "" + + "" + + "BlockBlob" + + "Hot" + + "true" + + "unlocked" + + "available" + + "true" + + "" + + "" + + "true" + + "" + + "" + + "" + + "" + + "hello/" + + "" + + "" + + "splits.txt" + + "" + + "Tue, 06 Jun 2023 08:33:51 GMT" + + "Mon, 19 Jun 2023 04:11:51 GMT" + + "0x8DB707B4F77B831" + + "67681" + + "text/plain" + + "" + + "" + + "" + + "If+Z0+KfGsXjvdpT1Z66NQ==" + + "" + + "" + + "BlockBlob" + + "Hot" + + "true" + + "unlocked" + + "available" + + "true" + + "" + + "" + + "hi" + + "%D0%91%d0%BB%D1%8E%D0%B7" + + "" + + "" + + "" + + "" + + "" + + ""; + BlobListResultSchema listResultSchema = getResultSchema(xmlResponseWithDelimiter); + List paths = listResultSchema.paths(); + Assertions.assertThat(paths.size()).isEqualTo(4); + Assertions.assertThat(paths.get(0).isDirectory()).isEqualTo(true); + Assertions.assertThat(paths.get(1).isDirectory()).isEqualTo(true); + Assertions.assertThat(paths.get(2).isDirectory()).isEqualTo(true); + Assertions.assertThat(paths.get(3).isDirectory()).isEqualTo(false); + } + + @Test + public void testEmptyBlobList() throws Exception { + String xmlResponse = "" + + "<" + + "EnumerationResults ServiceEndpoint=\"https://anujtestfns.blob.core.windows.net/\" ContainerName=\"manualtest\">" + + "abc/" + + "/" + + "" + + ""; + BlobListResultSchema listResultSchema = getResultSchema(xmlResponse); + List paths = listResultSchema.paths(); + Assertions.assertThat(paths.size()).isEqualTo(0); + } + + private static final ThreadLocal saxParserThreadLocal + = new ThreadLocal() { + @Override + public SAXParser initialValue() { + SAXParserFactory factory = SAXParserFactory.newInstance(); + factory.setNamespaceAware(true); + try { + return factory.newSAXParser(); + } catch (SAXException e) { + throw new RuntimeException("Unable to create SAXParser", e); + } catch (ParserConfigurationException e) { + throw new RuntimeException("Check parser configuration", e); + } + } + }; + + private BlobListResultSchema getResultSchema(String xmlResponse) throws Exception { + byte[] bytes = xmlResponse.getBytes(); + final InputStream stream = new ByteArrayInputStream(bytes);; + final SAXParser saxParser = saxParserThreadLocal.get(); + saxParser.reset(); + BlobListResultSchema listResultSchema = new BlobListResultSchema(); + saxParser.parse(stream, new BlobListXmlParser(listResultSchema, "https://sample.url")); + return listResultSchema; + } +} diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ListResultSchemaTest.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/TestDfsListJsonParser.java similarity index 99% rename from hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ListResultSchemaTest.java rename to hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/TestDfsListJsonParser.java index 3f6a4872c5d171..ec3b2d21aefeb7 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ListResultSchemaTest.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/TestDfsListJsonParser.java @@ -31,7 +31,7 @@ /** * Tests the JSON parsing for the listfilestatus response to ListResultSchema */ -public class ListResultSchemaTest { +public class TestDfsListJsonParser { /** * Test parsing a JSON which matches the properties in the ListResultSchema diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestAbfsClientHandler.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestAbfsClientHandler.java index 169398e6e99f8a..2a24fe3a9ba7b6 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestAbfsClientHandler.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestAbfsClientHandler.java @@ -45,5 +45,6 @@ public void testAbfsClientHandlerInitialization() throws Exception { Assertions.assertThat(clientHandler.getClient()).isInstanceOf(AbfsDfsClient.class); Assertions.assertThat(clientHandler.getClient(AbfsServiceType.DFS)).isInstanceOf(AbfsDfsClient.class); Assertions.assertThat(clientHandler.getClient(AbfsServiceType.BLOB)).isInstanceOf(AbfsBlobClient.class); + Assertions.assertThat(clientHandler.getIngressClient()).isInstanceOf(AbfsDfsClient.class); } } From 7a1ac02ef87f95d6cffe150012ba0790666a37fa Mon Sep 17 00:00:00 2001 From: Anuj Modi Date: Sat, 1 Feb 2025 04:32:36 -0800 Subject: [PATCH 02/13] Test Code Improvements --- .../fs/azurebfs/AbstractAbfsIntegrationTest.java | 14 ++++++++++---- .../azurebfs/ITestAzureBlobFileSystemAppend.java | 11 ++++------- .../fs/azurebfs/ITestAzureBlobFileSystemFlush.java | 2 +- .../azurebfs/services/ITestAbfsOutputStream.java | 4 ++-- 4 files changed, 17 insertions(+), 14 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java index 6d3bbad761c8fb..2d4d1f71428215 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java @@ -645,13 +645,19 @@ protected void assumeDfsServiceType() { } protected void assumeHnsEnabled() throws IOException { - Assume.assumeTrue("HNS-Enabled account must be used for this test", - getIsNamespaceEnabled(getFileSystem())); + assumeHnsEnabled("HNS-Enabled account must be used for this test"); + } + + protected void assumeHnsEnabled(String errorMessage) throws IOException { + Assume.assumeTrue(errorMessage, getIsNamespaceEnabled(getFileSystem())); } protected void assumeHnsDisabled() throws IOException { - Assume.assumeFalse("HNS-Enabled account must not be used for this test", - getIsNamespaceEnabled(getFileSystem())); + assumeHnsDisabled("HNS-Enabled account must not be used for this test"); + } + + protected void assumeHnsDisabled(String message) throws IOException { + Assume.assumeFalse(message, getIsNamespaceEnabled(getFileSystem())); } /** diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemAppend.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemAppend.java index 89043fa79504c4..dfbeb793cd1637 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemAppend.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemAppend.java @@ -341,9 +341,7 @@ public void testCreateAppendBlobOverBlobEndpointAppendOverDfs() @Test public void testCreateAppendBlobOverDfsEndpointAppendOverBlob() throws IOException, NoSuchFieldException, IllegalAccessException { - Assume.assumeTrue( - "FNS does not support append blob creation for DFS endpoint", - getIsNamespaceEnabled(getFileSystem())); + assumeHnsEnabled("FNS does not support append blob creation for DFS endpoint"); final AzureBlobFileSystem fs = Mockito.spy(getFileSystem()); AzureBlobFileSystemStore store = Mockito.spy(fs.getAbfsStore()); Mockito.doReturn(true).when(store).isAppendBlobKey(anyString()); @@ -777,7 +775,7 @@ public void testIntermittentAppendFailureToBeReported() throws Exception { Assume.assumeFalse("Not valid for APPEND BLOB", isAppendBlobEnabled()); try (AzureBlobFileSystem fs = Mockito.spy( (AzureBlobFileSystem) FileSystem.newInstance(getRawConfiguration()))) { - Assume.assumeTrue(!getIsNamespaceEnabled(fs)); + assumeHnsDisabled(); AzureBlobFileSystemStore store = Mockito.spy(fs.getAbfsStore()); assumeBlobServiceType(); @@ -982,8 +980,7 @@ public void testFlushSuccessWithConnectionResetOnResponseValidMd5() throws Excep // Create a spy of AzureBlobFileSystem try (AzureBlobFileSystem fs = Mockito.spy( (AzureBlobFileSystem) FileSystem.newInstance(getRawConfiguration()))) { - Assume.assumeTrue(!getIsNamespaceEnabled(fs)); - + assumeHnsDisabled(); // Create a spy of AzureBlobFileSystemStore AzureBlobFileSystemStore store = Mockito.spy(fs.getAbfsStore()); assumeBlobServiceType(); @@ -1080,7 +1077,7 @@ public void testFlushSuccessWithConnectionResetOnResponseInvalidMd5() throws Exc // Create a spy of AzureBlobFileSystem try (AzureBlobFileSystem fs = Mockito.spy( (AzureBlobFileSystem) FileSystem.newInstance(getRawConfiguration()))) { - Assume.assumeTrue(!getIsNamespaceEnabled(fs)); + assumeHnsDisabled(); // Create a spy of AzureBlobFileSystemStore AzureBlobFileSystemStore store = Mockito.spy(fs.getAbfsStore()); diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemFlush.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemFlush.java index 8833fe4bd9f493..57b7064a2c556f 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemFlush.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemFlush.java @@ -313,7 +313,7 @@ public void testTracingHeaderForAppendBlob() throws Exception { AzureBlobFileSystem fs1 = getFileSystem(); // AppendBlob not supported for FNS over DFS endpoint. if (!getIsNamespaceEnabled(fs1)) { - Assume.assumeTrue(getAbfsStore(fs1).getClient() instanceof AbfsBlobClient); + assumeBlobServiceType(); } Configuration config = new Configuration(this.getRawConfiguration()); config.set(FS_AZURE_APPEND_BLOB_KEY, "abfss:/"); diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestAbfsOutputStream.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestAbfsOutputStream.java index 3a7928d521035c..dff94aae117da2 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestAbfsOutputStream.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestAbfsOutputStream.java @@ -350,7 +350,7 @@ public void testNoNetworkCallsForFlush() throws Exception { AzureBlobFileSystem fs = Mockito.spy(getFileSystem()); Assume.assumeTrue(!getIsNamespaceEnabled(fs)); AzureBlobFileSystemStore store = Mockito.spy(fs.getAbfsStore()); - Assume.assumeTrue(store.getClient() instanceof AbfsBlobClient); + assumeBlobServiceType(); // Mock the clientHandler to return the blobClient when getBlobClient is called AbfsClientHandler clientHandler = Mockito.spy(store.getClientHandler()); @@ -393,7 +393,7 @@ public void testNoNetworkCallsForSecondFlush() throws Exception { AzureBlobFileSystem fs = Mockito.spy(getFileSystem()); Assume.assumeTrue(!getIsNamespaceEnabled(fs)); AzureBlobFileSystemStore store = Mockito.spy(fs.getAbfsStore()); - Assume.assumeTrue(store.getClient() instanceof AbfsBlobClient); + assumeBlobServiceType(); Assume.assumeFalse("Not valid for APPEND BLOB", isAppendBlobEnabled()); // Step 2: Mock the clientHandler to return the blobClient when getBlobClient is called From dff98e6e3ba0c852358aee0a56f467bec365f8c6 Mon Sep 17 00:00:00 2001 From: Anuj Modi Date: Sat, 1 Feb 2025 06:40:00 -0800 Subject: [PATCH 03/13] Fixing Failing Test --- .../fs/azurebfs/contract/TestDfsListJsonParser.java | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/TestDfsListJsonParser.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/TestDfsListJsonParser.java index ec3b2d21aefeb7..7e3b4930bcc5f7 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/TestDfsListJsonParser.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/TestDfsListJsonParser.java @@ -23,6 +23,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import org.junit.Test; +import org.apache.hadoop.fs.azurebfs.contracts.services.DfsListResultSchema; import org.apache.hadoop.fs.azurebfs.contracts.services.ListResultEntrySchema; import org.apache.hadoop.fs.azurebfs.contracts.services.ListResultSchema; @@ -62,8 +63,8 @@ public void testMatchingJSON() throws IOException { + "\"rw-r--r--\" } ] } "; final ObjectMapper objectMapper = new ObjectMapper(); - final ListResultSchema listResultSchema = objectMapper - .readValue(matchingJson, ListResultSchema.class); + final DfsListResultSchema listResultSchema = objectMapper + .readValue(matchingJson, DfsListResultSchema.class); assertThat(listResultSchema.paths().size()) .describedAs("Only one path is expected as present in the input JSON") @@ -123,8 +124,8 @@ public void testJSONWithUnknownFields() throws IOException { + "\"rw-r--r--\" } ] } "; final ObjectMapper objectMapper = new ObjectMapper(); - final ListResultSchema listResultSchema = objectMapper - .readValue(matchingJson, ListResultSchema.class); + final DfsListResultSchema listResultSchema = objectMapper + .readValue(matchingJson, DfsListResultSchema.class); assertThat(listResultSchema.paths().size()) .describedAs("Only one path is expected as present in the input JSON") From ba66d6f086ec54cf2cc882595ed1670cb8e82870 Mon Sep 17 00:00:00 2001 From: Anuj Modi Date: Sat, 1 Feb 2025 06:41:50 -0800 Subject: [PATCH 04/13] CheckStyle --- .../fs/azurebfs/ITestAzureBlobFileSystemChooseSAS.java | 1 - .../hadoop/fs/azurebfs/contract/TestBlobListXmlParser.java | 6 +++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemChooseSAS.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemChooseSAS.java index a0e6a3b5192767..dc40241a875769 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemChooseSAS.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemChooseSAS.java @@ -26,7 +26,6 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.azurebfs.constants.AbfsServiceType; import org.apache.hadoop.fs.azurebfs.contracts.exceptions.AzureBlobFileSystemException; import org.apache.hadoop.fs.azurebfs.contracts.exceptions.SASTokenProviderException; import org.apache.hadoop.fs.azurebfs.extensions.MockDelegationSASTokenProvider; diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/TestBlobListXmlParser.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/TestBlobListXmlParser.java index 4bfb5e5a14a2db..6dabd2467e87c2 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/TestBlobListXmlParser.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/TestBlobListXmlParser.java @@ -126,7 +126,7 @@ public void testEmptyBlobList() throws Exception { Assertions.assertThat(paths.size()).isEqualTo(0); } - private static final ThreadLocal saxParserThreadLocal + private static final ThreadLocal SAX_PARSER_THREAD_LOCAL = new ThreadLocal() { @Override public SAXParser initialValue() { @@ -144,8 +144,8 @@ public SAXParser initialValue() { private BlobListResultSchema getResultSchema(String xmlResponse) throws Exception { byte[] bytes = xmlResponse.getBytes(); - final InputStream stream = new ByteArrayInputStream(bytes);; - final SAXParser saxParser = saxParserThreadLocal.get(); + final InputStream stream = new ByteArrayInputStream(bytes); + final SAXParser saxParser = SAX_PARSER_THREAD_LOCAL.get(); saxParser.reset(); BlobListResultSchema listResultSchema = new BlobListResultSchema(); saxParser.parse(stream, new BlobListXmlParser(listResultSchema, "https://sample.url")); From 09a0c40cabf337239d1b78345e802450ab057b0c Mon Sep 17 00:00:00 2001 From: Anuj Modi Date: Sun, 2 Feb 2025 21:47:07 -0800 Subject: [PATCH 05/13] Updating tests Scripts --- .../dev-support/testrun-scripts/runtests.sh | 123 +++++++++++++++--- .../testrun-scripts/testsupport.sh | 19 +++ 2 files changed, 121 insertions(+), 21 deletions(-) diff --git a/hadoop-tools/hadoop-azure/dev-support/testrun-scripts/runtests.sh b/hadoop-tools/hadoop-azure/dev-support/testrun-scripts/runtests.sh index 400d0a23834a5f..a0039ced06d709 100755 --- a/hadoop-tools/hadoop-azure/dev-support/testrun-scripts/runtests.sh +++ b/hadoop-tools/hadoop-azure/dev-support/testrun-scripts/runtests.sh @@ -32,36 +32,87 @@ processCount=8 ## SECTION: TEST COMBINATION METHODS -runHNSOAuthTest() +runHNSOAuthDFSTest() { accountName=$(xmlstarlet sel -t -v '//property[name = "fs.azure.hnsTestAccountName"]/value' -n $azureTestXmlPath) PROPERTIES=("fs.azure.account.auth.type") VALUES=("OAuth") - triggerRun "HNS-OAuth" "$accountName" "$runTest" $processCount "$cleanUpTestContainers" + triggerRun "HNS-OAuth-DFS" "$accountName" "$runTest" $processCount "$cleanUpTestContainers" } -runHNSSharedKeyTest() +runHNSSharedKeyDFSTest() { accountName=$(xmlstarlet sel -t -v '//property[name = "fs.azure.hnsTestAccountName"]/value' -n $azureTestXmlPath) PROPERTIES=("fs.azure.account.auth.type") VALUES=("SharedKey") - triggerRun "HNS-SharedKey" "$accountName" "$runTest" $processCount "$cleanUpTestContainers" + triggerRun "HNS-SharedKey-DFS" "$accountName" "$runTest" $processCount "$cleanUpTestContainers" } -runNonHNSSharedKeyTest() +runNonHNSSharedKeyDFSTest() { accountName=$(xmlstarlet sel -t -v '//property[name = "fs.azure.nonHnsTestAccountName"]/value' -n $azureTestXmlPath) PROPERTIES=("fs.azure.account.auth.type") VALUES=("SharedKey") - triggerRun "NonHNS-SharedKey" "$accountName" "$runTest" $processCount "$cleanUpTestContainers" + triggerRun "NonHNS-SharedKey-DFS" "$accountName" "$runTest" $processCount "$cleanUpTestContainers" } -runAppendBlobHNSOAuthTest() +runAppendBlobHNSOAuthDFSTest() { accountName=$(xmlstarlet sel -t -v '//property[name = "fs.azure.hnsTestAccountName"]/value' -n $azureTestXmlPath) PROPERTIES=("fs.azure.account.auth.type" "fs.azure.test.appendblob.enabled") VALUES=("OAuth" "true") - triggerRun "AppendBlob-HNS-OAuth" "$accountName" "$runTest" $processCount "$cleanUpTestContainers" + triggerRun "AppendBlob-HNS-OAuth-DFS" "$accountName" "$runTest" $processCount "$cleanUpTestContainers" +} + +runNonHNSSharedKeyBlobTest() +{ + accountName=$(xmlstarlet sel -t -v '//property[name = "fs.azure.nonHnsTestAccountName"]/value' -n $azureTestXmlPath) + fnsBlobConfigFileCheck "$accountName" + PROPERTIES=("fs.azure.account.auth.type") + VALUES=("SharedKey") + triggerRun "NonHNS-SharedKey-Blob" "${accountName}_blob" "$runTest" $processCount "$cleanUpTestContainers" +} + +runNonHNSOAuthDFSTest() +{ + accountName=$(xmlstarlet sel -t -v '//property[name = "fs.azure.nonHnsTestAccountName"]/value' -n $azureTestXmlPath) + PROPERTIES=("fs.azure.account.auth.type") + VALUES=("OAuth") + triggerRun "NonHNS-OAuth-DFS" "$accountName" "$runTest" $processCount "$cleanUpTestContainers" +} + +runNonHNSOAuthBlobTest() +{ + accountName=$(xmlstarlet sel -t -v '//property[name = "fs.azure.nonHnsTestAccountName"]/value' -n $azureTestXmlPath) + fnsBlobConfigFileCheck "$accountName" + PROPERTIES=("fs.azure.account.auth.type") + VALUES=("OAuth") + triggerRun "NonHNS-OAuth-Blob" "${accountName}_blob" "$runTest" $processCount "$cleanUpTestContainers" +} + +runAppendBlobNonHNSOAuthBlobTest() +{ + accountName=$(xmlstarlet sel -t -v '//property[name = "fs.azure.nonHnsTestAccountName"]/value' -n $azureTestXmlPath) + fnsBlobConfigFileCheck "$accountName" + PROPERTIES=("fs.azure.account.auth.type" "fs.azure.test.appendblob.enabled") + VALUES=("OAuth" "true") + triggerRun "AppendBlob-NonHNS-OAuth-Blob" "${accountName}_blob" "$runTest" $processCount "$cleanUpTestContainers" +} + +runHNSOAuthDFSIngressBlobTest() +{ + accountName=$(xmlstarlet sel -t -v '//property[name = "fs.azure.hnsTestAccountName"]/value' -n $azureTestXmlPath) + PROPERTIES=("fs.azure.account.auth.type" "fs.azure.ingress.service.type") + VALUES=("OAuth" "blob") + triggerRun "HNS-Oauth-DFS-IngressBlob" "$accountName" "$runTest" $processCount "$cleanUpTestContainers" +} + +runNonHNSOAuthDFSIngressBlobTest() +{ + accountName=$(xmlstarlet sel -t -v '//property[name = "fs.azure.nonHnsTestAccountName"]/value' -n $azureTestXmlPath) + PROPERTIES=("fs.azure.account.auth.type" "fs.azure.ingress.service.type") + VALUES=("OAuth" "blob") + triggerRun "NonHNS-OAuth-DFS-IngressBlob" "$accountName" "$runTest" $processCount "$cleanUpTestContainers" } runTest=false @@ -130,23 +181,47 @@ done echo ' ' echo 'Set the active test combination to run the action:' -select combo in HNS-OAuth HNS-SharedKey nonHNS-SharedKey AppendBlob-HNS-OAuth AllCombinationsTestRun Quit +select combo in HNS-OAuth-DFS HNS-SharedKey-DFS NonHNS-SharedKey-DFS AppendBlob-HNS-OAuth-DFS NonHNS-SharedKey-Blob NonHNS-OAuth-DFS NonHNS-OAuth-Blob AppendBlob-NonHNS-OAuth-Blob HNS-Oauth-DFS-IngressBlob NonHNS-Oauth-DFS-IngressBlob AllCombinationsTestRun Quit do case $combo in - HNS-OAuth) - runHNSOAuthTest + HNS-OAuth-DFS) + runHNSOAuthDFSTest + break + ;; + HNS-SharedKey-DFS) + runHNSSharedKeyDFSTest + break + ;; + NonHNS-SharedKey-DFS) + runNonHNSSharedKeyDFSTest + break + ;; + AppendBlob-HNS-OAuth-DFS) + runAppendBlobHNSOAuthDFSTest + break + ;; + NonHNS-SharedKey-Blob) + runNonHNSSharedKeyBlobTest + break + ;; + NonHNS-OAuth-DFS) + runNonHNSOAuthDFSTest + break + ;; + NonHNS-OAuth-Blob) + runNonHNSOAuthBlobTest break ;; - HNS-SharedKey) - runHNSSharedKeyTest + AppendBlob-NonHNS-OAuth-Blob) + runAppendBlobNonHNSOAuthBlobTest break ;; - nonHNS-SharedKey) - runNonHNSSharedKeyTest + HNS-Oauth-DFS-IngressBlob) + runHNSOAuthDFSIngressBlobTest break ;; - AppendBlob-HNS-OAuth) - runAppendBlobHNSOAuthTest + NonHNS-Oauth-DFS-IngressBlob) + runNonHNSOAuthDFSIngressBlobTest break ;; AllCombinationsTestRun) @@ -155,10 +230,16 @@ do logOutput "ERROR: Invalid selection for SET_ACTIVE_TEST_CONFIG. This is applicable only for RUN_TEST." break fi - runHNSOAuthTest - runHNSSharedKeyTest - runNonHNSSharedKeyTest - runAppendBlobHNSOAuthTest ## Keep this as the last run scenario always + runHNSOAuthDFSTest + runHNSSharedKeyDFSTest + runNonHNSSharedKeyDFSTest + runAppendBlobHNSOAuthDFSTest + runNonHNSSharedKeyBlobTest + runNonHNSOAuthDFSTest + runNonHNSOAuthBlobTest + runAppendBlobNonHNSOAuthBlobTest + runHNSOAuthDFSIngressBlobTest + runNonHNSOAuthDFSIngressBlobTest break ;; Quit) diff --git a/hadoop-tools/hadoop-azure/dev-support/testrun-scripts/testsupport.sh b/hadoop-tools/hadoop-azure/dev-support/testrun-scripts/testsupport.sh index 8933b81145de1f..8862942a1119da 100644 --- a/hadoop-tools/hadoop-azure/dev-support/testrun-scripts/testsupport.sh +++ b/hadoop-tools/hadoop-azure/dev-support/testrun-scripts/testsupport.sh @@ -45,6 +45,25 @@ ENDTIME=$(date +%s) outputFormatOn="\033[0;95m" outputFormatOff="\033[0m" +# Function to check if the blob config file exists and create one if it doesn't. +fnsBlobConfigFileCheck() { + baseFileName=$1 + targetWord=".dfs." + replacementWord=".blob." + accountSettingsDir="src/test/resources/accountSettings/" + accountConfigFileSuffix="_settings.xml" + sourceFilePath="${accountSettingsDir}${baseFileName}${accountConfigFileSuffix}" + targetFilePath="${accountSettingsDir}${baseFileName}_blob${accountConfigFileSuffix}" + + if [ ! -f "$targetFilePath" ]; then + cp "$sourceFilePath" "$targetFilePath" + sed -i "s/$targetWord/$replacementWord/g" "$targetFilePath" + echo "File created and word replaced." + else + echo "File already exists." + fi +} + triggerRun() { echo ' ' From 50de6754571fc87fb05e1f02d6a1896262f23052 Mon Sep 17 00:00:00 2001 From: Anuj Modi Date: Sun, 2 Feb 2025 21:48:14 -0800 Subject: [PATCH 06/13] Checkstyle --- .../hadoop/fs/azurebfs/ITestAzureBlobFileSystemFlush.java | 2 -- .../hadoop/fs/azurebfs/contract/TestDfsListJsonParser.java | 1 - 2 files changed, 3 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemFlush.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemFlush.java index 57b7064a2c556f..b55032c5132a5d 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemFlush.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemFlush.java @@ -33,12 +33,10 @@ import org.apache.hadoop.fs.StreamCapabilities; import org.apache.hadoop.fs.azurebfs.constants.FSOperationType; import org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys; -import org.apache.hadoop.fs.azurebfs.services.AbfsBlobClient; import org.apache.hadoop.fs.azurebfs.services.AbfsOutputStream; import org.apache.hadoop.fs.azurebfs.utils.TracingHeaderValidator; import org.hamcrest.core.IsEqual; import org.hamcrest.core.IsNot; -import org.junit.Assume; import org.junit.Test; import org.apache.hadoop.fs.FSDataInputStream; diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/TestDfsListJsonParser.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/TestDfsListJsonParser.java index 7e3b4930bcc5f7..d62265ea57f78a 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/TestDfsListJsonParser.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/TestDfsListJsonParser.java @@ -25,7 +25,6 @@ import org.apache.hadoop.fs.azurebfs.contracts.services.DfsListResultSchema; import org.apache.hadoop.fs.azurebfs.contracts.services.ListResultEntrySchema; -import org.apache.hadoop.fs.azurebfs.contracts.services.ListResultSchema; import static org.assertj.core.api.Assertions.assertThat; From 61fbb733ee0e5147668ad1365e1f89054e45f619 Mon Sep 17 00:00:00 2001 From: Anuj Modi Date: Sun, 2 Feb 2025 22:39:08 -0800 Subject: [PATCH 07/13] Test Fix --- .../apache/hadoop/fs/azurebfs/AbfsConfiguration.java | 8 ++++---- .../hadoop/fs/azurebfs/services/AbfsPermission.java | 5 +++-- .../fs/azurebfs/services/ITestAbfsClientHandler.java | 10 ++++++++-- 3 files changed, 15 insertions(+), 8 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java index e0e245ba33b259..2ca0691aa44767 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java @@ -524,10 +524,10 @@ public boolean isDfsToBlobFallbackEnabled() { public void validateConfiguredServiceType(boolean isHNSEnabled) throws InvalidConfigurationValueException { // TODO: [FnsOverBlob][HADOOP-19179] Remove this check when FNS over Blob is ready. - if (getFsConfiguredServiceType() == AbfsServiceType.BLOB) { - throw new InvalidConfigurationValueException(FS_DEFAULT_NAME_KEY, - "Blob Endpoint Support not yet available"); - } +// if (getFsConfiguredServiceType() == AbfsServiceType.BLOB) { +// throw new InvalidConfigurationValueException(FS_DEFAULT_NAME_KEY, +// "Blob Endpoint Support not yet available"); +// } if (isHNSEnabled && getConfiguredServiceTypeForFNSAccounts() == AbfsServiceType.BLOB) { throw new InvalidConfigurationValueException( FS_AZURE_FNS_ACCOUNT_SERVICE_TYPE, "Service Type Cannot be BLOB for HNS Account"); diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsPermission.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsPermission.java index 9c44610ff592f3..bbeb68537a2ef0 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsPermission.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsPermission.java @@ -19,6 +19,7 @@ package org.apache.hadoop.fs.azurebfs.services; +import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.fs.permission.FsAction; import org.apache.hadoop.fs.permission.FsPermission; @@ -68,7 +69,7 @@ public boolean equals(Object obj) { * @return a permission object for the provided string representation */ public static AbfsPermission valueOf(final String abfsSymbolicPermission) { - if (abfsSymbolicPermission == null) { + if (StringUtils.isEmpty(abfsSymbolicPermission)) { return null; } @@ -100,7 +101,7 @@ public static AbfsPermission valueOf(final String abfsSymbolicPermission) { * extended ACL; otherwise false. */ public static boolean isExtendedAcl(final String abfsSymbolicPermission) { - if (abfsSymbolicPermission == null) { + if (StringUtils.isEmpty(abfsSymbolicPermission)) { return false; } diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestAbfsClientHandler.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestAbfsClientHandler.java index 2a24fe3a9ba7b6..986a0f525fd7c5 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestAbfsClientHandler.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestAbfsClientHandler.java @@ -42,9 +42,15 @@ public ITestAbfsClientHandler() throws Exception{ public void testAbfsClientHandlerInitialization() throws Exception { AzureBlobFileSystem fs = getFileSystem(); AbfsClientHandler clientHandler = fs.getAbfsStore().getClientHandler(); - Assertions.assertThat(clientHandler.getClient()).isInstanceOf(AbfsDfsClient.class); + if (getAbfsServiceType() == AbfsServiceType.DFS) { + Assertions.assertThat(clientHandler.getClient()).isInstanceOf(AbfsDfsClient.class); + Assertions.assertThat(clientHandler.getIngressClient()).isInstanceOf(AbfsDfsClient.class); + } else { + Assertions.assertThat(clientHandler.getClient()).isInstanceOf(AbfsBlobClient.class); + Assertions.assertThat(clientHandler.getIngressClient()).isInstanceOf(AbfsBlobClient.class); + } Assertions.assertThat(clientHandler.getClient(AbfsServiceType.DFS)).isInstanceOf(AbfsDfsClient.class); Assertions.assertThat(clientHandler.getClient(AbfsServiceType.BLOB)).isInstanceOf(AbfsBlobClient.class); - Assertions.assertThat(clientHandler.getIngressClient()).isInstanceOf(AbfsDfsClient.class); + } } From 27030e9c80fd561ee161b5fca1b3f80c19a2a608 Mon Sep 17 00:00:00 2001 From: Anuj Modi Date: Mon, 3 Feb 2025 09:45:57 -0800 Subject: [PATCH 08/13] Comments --- .../azurebfs/AbstractAbfsIntegrationTest.java | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java index 2d4d1f71428215..7e6789e0a139df 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java @@ -634,28 +634,56 @@ private String getAzcopyAbsolutePath(Path path) throws IOException { + accountName + FORWARD_SLASH + fileSystemName + pathFromContainerRoot; } + /** + * Utility method to assume that the test is running against a Blob service. + * Otherwise, the test will be skipped. + */ protected void assumeBlobServiceType() { Assume.assumeTrue("Blob service type is required for this test", getAbfsServiceType() == AbfsServiceType.BLOB); } + /** + * Utility method to assume that the test is running against a DFS service. + * Otherwise, the test will be skipped. + */ protected void assumeDfsServiceType() { Assume.assumeTrue("DFS service type is required for this test", getAbfsServiceType() == AbfsServiceType.DFS); } + /** + * Utility method to assume that the test is running against a HNS Enabled account. + * Otherwise, the test will be skipped. + * @throws IOException if an error occurs while checking the account type. + */ protected void assumeHnsEnabled() throws IOException { assumeHnsEnabled("HNS-Enabled account must be used for this test"); } + /** + * Utility method to assume that the test is running against a HNS Enabled account. + * @param errorMessage error message to be displayed if the test is skipped. + * @throws IOException if an error occurs while checking the account type. + */ protected void assumeHnsEnabled(String errorMessage) throws IOException { Assume.assumeTrue(errorMessage, getIsNamespaceEnabled(getFileSystem())); } + /** + * Utility method to assume that the test is running against a HNS Disabled account. + * Otherwise, the test will be skipped. + * @throws IOException if an error occurs while checking the account type. + */ protected void assumeHnsDisabled() throws IOException { assumeHnsDisabled("HNS-Enabled account must not be used for this test"); } + /** + * Utility method to assume that the test is running against a HNS Disabled account. + * @param message error message to be displayed if the test is skipped. + * @throws IOException if an error occurs while checking the account type. + */ protected void assumeHnsDisabled(String message) throws IOException { Assume.assumeFalse(message, getIsNamespaceEnabled(getFileSystem())); } From a13ce372d63fb0c8e9b8870c6dca7b49164367d4 Mon Sep 17 00:00:00 2001 From: Anuj Modi Date: Mon, 3 Feb 2025 10:01:09 -0800 Subject: [PATCH 09/13] Javadocs --- .../azurebfs/ITestAzureBlobFileSystemFileStatus.java | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemFileStatus.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemFileStatus.java index 9bb7d1cc7aef61..87b1ea4321d416 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemFileStatus.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemFileStatus.java @@ -258,6 +258,11 @@ public void testListStatusIsCalledForImplicitPathOnBlobEndpoint() throws Excepti Mockito.verify(abfsClient, Mockito.times(1)).listPath(any(), eq(false), eq(1), any(), any(), eq(false)); } + /** + * Verifies the file status indicates a file present in the path. + * @param fileStatus + * @param isDir + */ private void verifyFileStatus(FileStatus fileStatus, boolean isDir) { Assertions.assertThat(fileStatus).isNotNull(); if (isDir) { @@ -269,6 +274,11 @@ private void verifyFileStatus(FileStatus fileStatus, boolean isDir) { assertPathDns(fileStatus.getPath()); } + /** + * Verifies the file not found exception is thrown with the expected message. + * @param ex + * @param key + */ private void verifyFileNotFound(FileNotFoundException ex, String key) { Assertions.assertThat(ex).isNotNull(); Assertions.assertThat(ex.getMessage()).contains(key); From d7d086faa1d25afc1ab4d72061d8c66c135be534 Mon Sep 17 00:00:00 2001 From: Anuj Modi Date: Mon, 3 Feb 2025 20:08:38 -0800 Subject: [PATCH 10/13] Comments --- .../hadoop/fs/azurebfs/AbfsConfiguration.java | 5 ---- .../ITestAzureBlobFileSystemAttributes.java | 24 +++++++++++++++++-- ...ITestAzureBlobFileSystemInitAndCreate.java | 16 ------------- .../services/ITestAbfsClientHandler.java | 8 +++++-- 4 files changed, 28 insertions(+), 25 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java index a0db47859870ad..e0cb36201065d9 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java @@ -550,11 +550,6 @@ public boolean isDfsToBlobFallbackEnabled() { */ public void validateConfiguredServiceType(boolean isHNSEnabled) throws InvalidConfigurationValueException { - // TODO: [FnsOverBlob][HADOOP-19179] Remove this check when FNS over Blob is ready. -// if (getFsConfiguredServiceType() == AbfsServiceType.BLOB) { -// throw new InvalidConfigurationValueException(FS_DEFAULT_NAME_KEY, -// "Blob Endpoint Support not yet available"); -// } if (isHNSEnabled && getConfiguredServiceTypeForFNSAccounts() == AbfsServiceType.BLOB) { throw new InvalidConfigurationValueException( FS_AZURE_FNS_ACCOUNT_SERVICE_TYPE, "Service Type Cannot be BLOB for HNS Account"); diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemAttributes.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemAttributes.java index f6d1dc2674b4b4..3e0959d5dd6a09 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemAttributes.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemAttributes.java @@ -33,6 +33,7 @@ import org.apache.hadoop.fs.azurebfs.utils.TracingHeaderValidator; import static java.net.HttpURLConnection.HTTP_BAD_REQUEST; +import static java.net.HttpURLConnection.HTTP_NOT_FOUND; import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.ROOT_PATH; import static org.apache.hadoop.test.LambdaTestUtils.intercept; @@ -95,6 +96,10 @@ public void testSetGetXAttrReplace() throws Exception { decodedAttribute2); } + /** + * Test get and set xattr fails fine on root. + * @throws Exception if test fails + */ @Test public void testGetSetXAttrOnRoot() throws Exception { AzureBlobFileSystem fs = getFileSystem(); @@ -118,6 +123,11 @@ public void testGetSetXAttrOnRoot() throws Exception { .describedAs("SetXAttr() on root should fail with Bad Request") .isEqualTo(HTTP_BAD_REQUEST); } + + /** + * Test get and set xattr works fine on marker blobs for directory. + * @throws Exception if test fails + */ @Test public void testGetSetXAttrOnExplicitDir() throws Exception { AzureBlobFileSystem fs = getFileSystem(); @@ -128,12 +138,18 @@ public void testGetSetXAttrOnExplicitDir() throws Exception { // Assert that the folder is now explicit DirectoryStateHelper.isExplicitDirectory(testPath, fs, getTestTracingContext(fs, true)); } + + /** + * Test get and set xattr fails fine on non-existing path. + * @throws Exception if test fails + */ @Test public void testGetSetXAttrOnNonExistingPath() throws Exception { AzureBlobFileSystem fs = getFileSystem(); final Path testPath = path(getMethodName()); - intercept( - FileNotFoundException.class, () -> testGetSetXAttrHelper(fs, testPath)); + intercept(FileNotFoundException.class, String.valueOf(HTTP_NOT_FOUND), + "get/set XAttr() should fail for non-existing files", + () -> testGetSetXAttrHelper(fs, testPath)); } /** @@ -163,6 +179,10 @@ public void testSetXAttrMultipleOperations() throws Exception { assertAttributeEqual(rv, attributeValue, decodedAttributeValue); } + /** + * Test get and set xattr works fine on implicit directory and ends up creating marker blobs. + * @throws Exception if test fails + */ @Test public void testGetSetXAttrOnImplicitDir() throws Exception { assumeBlobServiceType(); diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemInitAndCreate.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemInitAndCreate.java index ca91812296cdc7..014023243534c4 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemInitAndCreate.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemInitAndCreate.java @@ -115,22 +115,6 @@ public void testNoGetAclCallOnHnsConfigPresence() throws Exception { .getAclStatus(Mockito.anyString(), any(TracingContext.class)); } - // TODO: [FnsOverBlob][HADOOP-19179] Remove this test case once Blob Endpoint Support is enabled. - @Test - public void testFileSystemInitFailsWithBlobEndpointUrl() throws Exception { - Configuration configuration = new Configuration(getRawConfiguration()); - String defaultUri = configuration.get(FS_DEFAULT_NAME_KEY); - String accountKey = configuration.get( - accountProperty(FS_AZURE_ACCOUNT_KEY_PROPERTY_NAME, getAccountName()), - configuration.get(FS_AZURE_ACCOUNT_KEY_PROPERTY_NAME)); - configuration.set(FS_AZURE_ACCOUNT_KEY_PROPERTY_NAME, - accountKey.replace(ABFS_DFS_DOMAIN_NAME, ABFS_BLOB_DOMAIN_NAME)); - String blobUri = defaultUri.replace(ABFS_DFS_DOMAIN_NAME, ABFS_BLOB_DOMAIN_NAME); - intercept(InvalidConfigurationValueException.class, - "Blob Endpoint Support not yet available", () -> - FileSystem.newInstance(new Path(blobUri).toUri(), configuration)); - } - @Test public void testFileSystemInitFailsIfNotAbleToDetermineAccountType() throws Exception { AzureBlobFileSystem fs = ((AzureBlobFileSystem) FileSystem.newInstance( diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestAbfsClientHandler.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestAbfsClientHandler.java index 5d9ccb8bd91871..85e20fb274e2c5 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestAbfsClientHandler.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestAbfsClientHandler.java @@ -44,10 +44,14 @@ public void testAbfsClientHandlerInitialization() throws Exception { AbfsClientHandler clientHandler = fs.getAbfsStore().getClientHandler(); if (getAbfsServiceType() == AbfsServiceType.DFS) { Assertions.assertThat(clientHandler.getClient()).isInstanceOf(AbfsDfsClient.class); - Assertions.assertThat(clientHandler.getIngressClient()).isInstanceOf(AbfsDfsClient.class); } else { Assertions.assertThat(clientHandler.getClient()).isInstanceOf(AbfsBlobClient.class); - Assertions.assertThat(clientHandler.getIngressClient()).isInstanceOf(AbfsBlobClient.class); + } + if (getIngressServiceType() == AbfsServiceType.DFS) { + Assertions.assertThat(clientHandler.getIngressClient()).isInstanceOf(AbfsDfsClient.class); + } else { + Assertions.assertThat(clientHandler.getIngressClient()) + .isInstanceOf(AbfsBlobClient.class); } Assertions.assertThat(clientHandler.getClient(AbfsServiceType.DFS)).isInstanceOf(AbfsDfsClient.class); Assertions.assertThat(clientHandler.getClient(AbfsServiceType.BLOB)).isInstanceOf(AbfsBlobClient.class); From 2f4a0c640b2c8a87f5331cd7454acf81a29052ac Mon Sep 17 00:00:00 2001 From: Anuj Modi Date: Mon, 3 Feb 2025 20:55:01 -0800 Subject: [PATCH 11/13] Code Improvements --- .../hadoop/fs/azurebfs/ITestAzureBlobFileSystemDelete.java | 2 +- .../hadoop/fs/azurebfs/ITestAzureBlobFileSystemListStatus.java | 2 +- .../hadoop/fs/azurebfs/services/ITestAbfsPaginatedDelete.java | 2 +- .../fs/azurebfs/services/TestAbfsRenameRetryRecovery.java | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemDelete.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemDelete.java index f7fb57556884d8..c596040b363118 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemDelete.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemDelete.java @@ -240,7 +240,7 @@ public void testDeleteIdempotencyTriggerHttp404() throws Exception { * endpoint, the orchestration would be done by the client. The idempotency * issue would not happen for blob endpoint. */ - Assume.assumeTrue(getAbfsServiceType() == AbfsServiceType.DFS); + assumeDfsServiceType(); AbfsClient client = ITestAbfsClient.createTestClientFromCurrentContext( fs.getAbfsStore().getClient(), this.getConfiguration()); diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemListStatus.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemListStatus.java index 73b4590fc1470b..dc9c0e25cc1d31 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemListStatus.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemListStatus.java @@ -124,7 +124,7 @@ public Void call() throws Exception { */ @Test public void testListPathTracingContext() throws Exception { - Assume.assumeTrue(getAbfsServiceType() == AbfsServiceType.DFS); + assumeDfsServiceType(); final AzureBlobFileSystem fs = getFileSystem(); final AzureBlobFileSystem spiedFs = Mockito.spy(fs); final AzureBlobFileSystemStore spiedStore = Mockito.spy(fs.getAbfsStore()); diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestAbfsPaginatedDelete.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestAbfsPaginatedDelete.java index b9de2a5d18bac7..ecb96df27266b8 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestAbfsPaginatedDelete.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestAbfsPaginatedDelete.java @@ -168,7 +168,7 @@ public void testRecursiveDeleteWithPagination() throws Exception { */ @Test public void testNonRecursiveDeleteWithPagination() throws Exception { - Assume.assumeTrue(getAbfsServiceType() == AbfsServiceType.DFS); + assumeDfsServiceType(); testNonRecursiveDeleteWithPaginationInternal(true); testNonRecursiveDeleteWithPaginationInternal(false); } diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsRenameRetryRecovery.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsRenameRetryRecovery.java index 81661d9fa383f4..ce2356f65809a2 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsRenameRetryRecovery.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsRenameRetryRecovery.java @@ -440,7 +440,7 @@ public void testRenameRecoveryUnsupportedForFlatNamespace() throws Exception { Assume.assumeTrue(!isNamespaceEnabled); // In DFS endpoint, renamePath is O(1) API call and idempotency issue can happen. // For blob endpoint, client orchestrates the rename operation. - Assume.assumeTrue(getAbfsServiceType() == AbfsServiceType.DFS); + assumeDfsServiceType(); AzureBlobFileSystem fs = getFileSystem(); AzureBlobFileSystemStore abfsStore = fs.getAbfsStore(); TracingContext testTracingContext = getTestTracingContext(fs, false); From 9ef0bcf0ee9f4b20397a25986792d2a13b5f92f9 Mon Sep 17 00:00:00 2001 From: Anuj Modi Date: Mon, 3 Feb 2025 22:25:34 -0800 Subject: [PATCH 12/13] Removing Unused Imports --- .../fs/azurebfs/ITestAzureBlobFileSystemInitAndCreate.java | 7 ------- 1 file changed, 7 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemInitAndCreate.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemInitAndCreate.java index 014023243534c4..a0796c1b36fd67 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemInitAndCreate.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemInitAndCreate.java @@ -20,7 +20,6 @@ import java.io.FileNotFoundException; -import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -32,7 +31,6 @@ import org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys; import org.apache.hadoop.fs.azurebfs.contracts.exceptions.AbfsRestOperationException; import org.apache.hadoop.fs.azurebfs.contracts.exceptions.AzureBlobFileSystemException; -import org.apache.hadoop.fs.azurebfs.contracts.exceptions.InvalidConfigurationValueException; import org.apache.hadoop.fs.azurebfs.contracts.exceptions.TrileanConversionException; import org.apache.hadoop.fs.azurebfs.enums.Trilean; import org.apache.hadoop.fs.azurebfs.services.AbfsClient; @@ -40,12 +38,7 @@ import org.apache.hadoop.fs.azurebfs.utils.TracingContext; import static java.net.HttpURLConnection.HTTP_UNAVAILABLE; -import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY; import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ACCOUNT_IS_HNS_ENABLED; -import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ACCOUNT_KEY_PROPERTY_NAME; -import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.accountProperty; -import static org.apache.hadoop.fs.azurebfs.constants.FileSystemUriSchemes.ABFS_BLOB_DOMAIN_NAME; -import static org.apache.hadoop.fs.azurebfs.constants.FileSystemUriSchemes.ABFS_DFS_DOMAIN_NAME; import static org.apache.hadoop.test.LambdaTestUtils.intercept; import static org.mockito.ArgumentMatchers.any; From d2b4125a53e0a0cb43c4d8d036a6251cc07ff2a2 Mon Sep 17 00:00:00 2001 From: Anuj Modi Date: Mon, 3 Feb 2025 23:05:13 -0800 Subject: [PATCH 13/13] Removing Unused Imports --- .../hadoop/fs/azurebfs/ITestAzureBlobFileSystemDelete.java | 1 - .../hadoop/fs/azurebfs/services/ITestAbfsPaginatedDelete.java | 1 - .../hadoop/fs/azurebfs/services/TestAbfsRenameRetryRecovery.java | 1 - 3 files changed, 3 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemDelete.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemDelete.java index c596040b363118..60bc39979d9d64 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemDelete.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemDelete.java @@ -37,7 +37,6 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.azurebfs.constants.AbfsServiceType; import org.apache.hadoop.fs.azurebfs.constants.FSOperationType; import org.apache.hadoop.fs.azurebfs.contracts.exceptions.AbfsRestOperationException; import org.apache.hadoop.fs.azurebfs.contracts.services.StorageErrorResponseSchema; diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestAbfsPaginatedDelete.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestAbfsPaginatedDelete.java index ecb96df27266b8..392443caec3d12 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestAbfsPaginatedDelete.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestAbfsPaginatedDelete.java @@ -34,7 +34,6 @@ import org.apache.hadoop.fs.azurebfs.AbstractAbfsIntegrationTest; import org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem; import org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants; -import org.apache.hadoop.fs.azurebfs.constants.AbfsServiceType; import org.apache.hadoop.fs.azurebfs.contracts.exceptions.AbfsRestOperationException; import org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider; import org.apache.hadoop.fs.azurebfs.utils.AclTestHelpers; diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsRenameRetryRecovery.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsRenameRetryRecovery.java index ce2356f65809a2..03a3089923fd56 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsRenameRetryRecovery.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsRenameRetryRecovery.java @@ -39,7 +39,6 @@ import org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem; import org.apache.hadoop.fs.azurebfs.AzureBlobFileSystemStore; import org.apache.hadoop.fs.azurebfs.commit.ResilientCommitByRename; -import org.apache.hadoop.fs.azurebfs.constants.AbfsServiceType; import org.apache.hadoop.fs.azurebfs.constants.HttpHeaderConfigurations; import org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys; import org.apache.hadoop.fs.azurebfs.contracts.exceptions.AbfsRestOperationException;