From 22a774ae4d6dd8036d53bb705728b03bb92cdb39 Mon Sep 17 00:00:00 2001 From: Ankit Jain Date: Mon, 9 Sep 2019 17:37:16 -0700 Subject: [PATCH 1/2] PHOENIX-5274: ConnectionQueryServiceImpl#ensureNamespaceCreated and ensureTableCreated should use HBase APIs that do not require ADMIN permissions for existence checks (Use hbaseAdmin listNamespaces API rather than getNamespaceDescriptor) --- .../phoenix/end2end/CreateSchemaIT.java | 5 +-- .../apache/phoenix/end2end/DropSchemaIT.java | 14 +++----- .../SystemCatalogCreationOnConnectionIT.java | 9 ++--- .../query/ConnectionQueryServicesImpl.java | 22 +++--------- .../org/apache/phoenix/util/ServerUtil.java | 26 +++++++++++++- .../apache/phoenix/util/ServerUtilTest.java | 36 +++++++++++++++++++ 6 files changed, 76 insertions(+), 36 deletions(-) create mode 100644 phoenix-core/src/test/java/org/apache/phoenix/util/ServerUtilTest.java diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/CreateSchemaIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/CreateSchemaIT.java index a05d7023382..d1d0efb8494 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/CreateSchemaIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/CreateSchemaIT.java @@ -18,7 +18,7 @@ package org.apache.phoenix.end2end; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import java.sql.Connection; @@ -31,6 +31,7 @@ import org.apache.phoenix.jdbc.PhoenixConnection; import org.apache.phoenix.query.QueryServices; import org.apache.phoenix.schema.SchemaAlreadyExistsException; +import org.apache.phoenix.util.ServerUtil; import org.apache.phoenix.util.PropertiesUtil; import org.apache.phoenix.util.SchemaUtil; import org.apache.phoenix.util.TestUtil; @@ -47,7 +48,7 @@ public void testCreateSchema() throws Exception { try (Connection conn = DriverManager.getConnection(getUrl(), props); Admin admin = conn.unwrap(PhoenixConnection.class).getQueryServices().getAdmin();) { conn.createStatement().execute(ddl); - assertNotNull(admin.getNamespaceDescriptor(schemaName)); + assertTrue(ServerUtil.isHbaseNamespaceAvailable(admin, schemaName)); } try (Connection conn = DriverManager.getConnection(getUrl(), props)) { conn.createStatement().execute(ddl); diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/DropSchemaIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/DropSchemaIT.java index 97ab29afee2..9eb984d63d3 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/DropSchemaIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/DropSchemaIT.java @@ -18,7 +18,7 @@ package org.apache.phoenix.end2end; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import java.sql.Connection; @@ -30,12 +30,12 @@ import java.util.Properties; import org.apache.hadoop.hbase.NamespaceDescriptor; -import org.apache.hadoop.hbase.NamespaceNotFoundException; import org.apache.hadoop.hbase.client.Admin; import org.apache.phoenix.exception.SQLExceptionCode; import org.apache.phoenix.jdbc.PhoenixConnection; import org.apache.phoenix.query.QueryServices; import org.apache.phoenix.schema.SchemaNotFoundException; +import org.apache.phoenix.util.ServerUtil; import org.apache.phoenix.util.ReadOnlyProps; import org.apache.phoenix.util.SchemaUtil; import org.junit.BeforeClass; @@ -92,22 +92,18 @@ public void testDropSchema() throws Exception { } catch (SQLException e) { assertEquals(e.getErrorCode(), SQLExceptionCode.CANNOT_MUTATE_SCHEMA.getErrorCode()); } - assertNotNull(admin.getNamespaceDescriptor(normalizeSchemaIdentifier)); + assertTrue(ServerUtil.isHbaseNamespaceAvailable(admin, normalizeSchemaIdentifier)); conn.createStatement().execute("DROP TABLE " + schema + "." + tableName); conn.createStatement().execute(ddl); - try { - admin.getNamespaceDescriptor(normalizeSchemaIdentifier); + if(ServerUtil.isHbaseNamespaceAvailable(admin, normalizeSchemaIdentifier)) fail(); - } catch (NamespaceNotFoundException ne) { - // expected - } conn.createStatement().execute("DROP SCHEMA IF EXISTS " + schema); admin.createNamespace(NamespaceDescriptor.create(normalizeSchemaIdentifier).build()); conn.createStatement().execute("DROP SCHEMA IF EXISTS " + schema); - assertNotNull(admin.getNamespaceDescriptor(normalizeSchemaIdentifier)); + assertTrue(ServerUtil.isHbaseNamespaceAvailable(admin, normalizeSchemaIdentifier)); conn.createStatement().execute("CREATE SCHEMA " + schema); conn.createStatement().execute("DROP SCHEMA " + schema); try { diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/SystemCatalogCreationOnConnectionIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/SystemCatalogCreationOnConnectionIT.java index de047a364dc..748e75baeaa 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/SystemCatalogCreationOnConnectionIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/SystemCatalogCreationOnConnectionIT.java @@ -41,7 +41,6 @@ import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HTableDescriptor; -import org.apache.hadoop.hbase.NamespaceNotFoundException; import org.apache.hadoop.hbase.TableName; import org.apache.phoenix.coprocessor.MetaDataProtocol; import org.apache.phoenix.exception.SQLExceptionCode; @@ -55,6 +54,7 @@ import org.apache.phoenix.query.QueryConstants; import org.apache.phoenix.query.QueryServices; import org.apache.phoenix.query.QueryServicesTestImpl; +import org.apache.phoenix.util.ServerUtil; import org.apache.phoenix.util.ReadOnlyProps; import org.apache.phoenix.util.UpgradeUtil; import org.junit.After; @@ -456,12 +456,7 @@ private Set getHBaseTables() throws IOException { // Check if the SYSTEM namespace has been created private boolean isSystemNamespaceCreated() throws IOException { - try { - testUtil.getAdmin().getNamespaceDescriptor(SYSTEM_CATALOG_SCHEMA); - } catch (NamespaceNotFoundException ex) { - return false; - } - return true; + return ServerUtil.isHbaseNamespaceAvailable(testUtil.getConnection().getAdmin(), SYSTEM_CATALOG_SCHEMA); } /** diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java index e5c935d70e8..3b6e10a46b6 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java @@ -1150,15 +1150,9 @@ private boolean allowOnlineTableSchemaUpdate() { boolean ensureNamespaceCreated(String schemaName) throws SQLException { SQLException sqlE = null; boolean createdNamespace = false; - try (Admin admin = getAdmin()) { - NamespaceDescriptor namespaceDescriptor = null; - try { - namespaceDescriptor = admin.getNamespaceDescriptor(schemaName); - } catch (NamespaceNotFoundException ignored) { - - } - if (namespaceDescriptor == null) { - namespaceDescriptor = NamespaceDescriptor.create(schemaName).build(); + try (Admin admin = connection.getAdmin()){ + if (!ServerUtil.isHbaseNamespaceAvailable(admin, schemaName)) { + NamespaceDescriptor namespaceDescriptor = NamespaceDescriptor.create(schemaName).build(); admin.createNamespace(namespaceDescriptor); createdNamespace = true; } @@ -5262,17 +5256,11 @@ public MetaDataResponse call(MetaDataService instance) throws IOException { private void ensureNamespaceDropped(String schemaName) throws SQLException { SQLException sqlE = null; - try (Admin admin = getAdmin()) { + try (Admin admin = connection.getAdmin()) { final String quorum = ZKConfig.getZKQuorumServersString(config); final String znode = this.props.get(HConstants.ZOOKEEPER_ZNODE_PARENT); LOGGER.debug("Found quorum: " + quorum + ":" + znode); - boolean nameSpaceExists = true; - try { - admin.getNamespaceDescriptor(schemaName); - } catch (org.apache.hadoop.hbase.NamespaceNotFoundException e) { - nameSpaceExists = false; - } - if (nameSpaceExists) { + if (ServerUtil.isHbaseNamespaceAvailable(admin, schemaName)) { admin.deleteNamespace(schemaName); } } catch (IOException e) { diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/ServerUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/ServerUtil.java index e308b3dd498..86176ad72fb 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/util/ServerUtil.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/util/ServerUtil.java @@ -37,6 +37,7 @@ import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.NotServingRegionException; import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.client.Connection; import org.apache.hadoop.hbase.client.RetriesExhaustedWithDetailsException; import org.apache.hadoop.hbase.client.Table; @@ -405,5 +406,28 @@ public static Configuration getNoRetriesIndexWriterConfigurationWithCustomThread clonedConf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 1); return clonedConf; - } + } + + /** + * Returns true if HBase namespace exists, else returns false + * @param admin HbaseAdmin Object + * @param schemaName Phoenix schema name for which we check existence of the HBase namespace + * @return true if the HBase namespace exists, else returns false + * @throws SQLException If there is an exception checking the HBase namespace + */ + public static boolean isHbaseNamespaceAvailable(Admin admin, String schemaName) throws IOException{ + boolean namespaceExists = false; + try{ + String[] hbaseNamespaces = admin.listNamespaces(); + for(String namespace : hbaseNamespaces){ + if(namespace.equals(schemaName)){ + namespaceExists = true; + break; + } + } + } catch (IOException e) { + throw e; + } + return namespaceExists; + } } diff --git a/phoenix-core/src/test/java/org/apache/phoenix/util/ServerUtilTest.java b/phoenix-core/src/test/java/org/apache/phoenix/util/ServerUtilTest.java new file mode 100644 index 00000000000..33e57b2cab5 --- /dev/null +++ b/phoenix-core/src/test/java/org/apache/phoenix/util/ServerUtilTest.java @@ -0,0 +1,36 @@ +package org.apache.phoenix.util; + +import org.apache.hadoop.hbase.client.Admin; +import org.junit.Test; +import org.mockito.Mockito; + +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertFalse; + +public class ServerUtilTest { + + String existingNamespaceOne = "existingNamespaceOne"; + String existingNamespaceTwo = "existingNamespaceTwo"; + String nonExistingNamespace = "nonExistingNamespace"; + + String[] namespaces = { existingNamespaceOne, existingNamespaceTwo }; + + @Test + public void testIsHbaseNamespaceAvailableWithExistingNamespace() throws Exception { + Admin mockAdmin = getMockedAdmin(); + assertTrue(ServerUtil.isHbaseNamespaceAvailable(mockAdmin, existingNamespaceOne)); + } + + @Test + public void testIsHbaseNamespaceAvailableWithNonExistingNamespace() throws Exception{ + Admin mockAdmin = getMockedAdmin(); + assertFalse(ServerUtil.isHbaseNamespaceAvailable(mockAdmin,nonExistingNamespace)); + } + + private Admin getMockedAdmin() throws Exception { + Admin mockAdmin = Mockito.mock(Admin.class); + Mockito.when(mockAdmin.listNamespaces()).thenReturn(namespaces); + return mockAdmin; + } + +} \ No newline at end of file From 1cc6d31fc35ee66e8c7fdcbdaf72d69d41d7985f Mon Sep 17 00:00:00 2001 From: Ankit Jain Date: Tue, 4 May 2021 21:07:30 -0700 Subject: [PATCH 2/2] PHOENIX-6437: Parent-Child Delete marker should get replicated via SystemCatalogWalEntryFilter --- .../SystemCatalogWALEntryFilterIT.java | 184 +++++++++++++++--- .../SystemCatalogWALEntryFilter.java | 25 +-- 2 files changed, 170 insertions(+), 39 deletions(-) diff --git a/phoenix-core/src/it/java/org/apache/phoenix/replication/SystemCatalogWALEntryFilterIT.java b/phoenix-core/src/it/java/org/apache/phoenix/replication/SystemCatalogWALEntryFilterIT.java index 6cd8b781209..a590d874fd8 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/replication/SystemCatalogWALEntryFilterIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/replication/SystemCatalogWALEntryFilterIT.java @@ -29,6 +29,8 @@ import org.apache.hadoop.hbase.client.ConnectionFactory; import org.apache.hadoop.hbase.client.Get; import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.ResultScanner; +import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.client.Table; import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import org.apache.hadoop.hbase.replication.ChainWALEntryFilter; @@ -37,6 +39,7 @@ import org.apache.hadoop.hbase.wal.WALEdit; import org.apache.hadoop.hbase.wal.WALKeyImpl; import org.apache.phoenix.end2end.ParallelStatsDisabledIT; +import org.apache.phoenix.hbase.index.wal.IndexedKeyValue; import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData; import org.apache.phoenix.mapreduce.util.ConnectionUtil; import org.apache.phoenix.schema.PTable; @@ -71,8 +74,8 @@ public class SystemCatalogWALEntryFilterIT extends ParallelStatsDisabledIT { + NONTENANT_VIEW_NAME + "(" + VIEW_COLUMN_NAME + " varchar) AS SELECT * FROM " + TestUtil.ENTITY_HISTORY_TABLE_NAME + " WHERE OLD_VALUE like 'E%'"; - private static final String DROP_TENANT_VIEW_SQL = "DROP VIEW IF EXISTS " + TENANT_VIEW_NAME; - private static final String DROP_NONTENANT_VIEW_SQL = "DROP VIEW IF EXISTS " + NONTENANT_VIEW_NAME; + private static final String DROP_TENANT_VIEW_SQL = "DROP VIEW IF EXISTS " + SCHEMA_NAME + "." + TENANT_VIEW_NAME; + private static final String DROP_NONTENANT_VIEW_SQL = "DROP VIEW IF EXISTS " + SCHEMA_NAME + "." + NONTENANT_VIEW_NAME; private static PTable catalogTable; private static PTable childLinkTable; private static WALKeyImpl walKeyCatalog = null; @@ -100,24 +103,13 @@ public static synchronized void setup() throws Exception { PhoenixDatabaseMetaData.SYSTEM_CHILD_LINK_NAME), 0, 0, uuid); }; Assert.assertNotNull(catalogTable); - try (java.sql.Connection connection = - ConnectionUtil.getInputConnection(getUtility().getConfiguration(), new Properties())) { - connection.createStatement().execute(CREATE_NONTENANT_VIEW_SQL); - }; + createNonTenantView(); } @AfterClass public static synchronized void tearDown() throws Exception { - Properties tenantProperties = new Properties(); - tenantProperties.setProperty("TenantId", TENANT_ID); - try (java.sql.Connection connection = - ConnectionUtil.getInputConnection(getUtility().getConfiguration(), tenantProperties)) { - connection.createStatement().execute(DROP_TENANT_VIEW_SQL); - } - try (java.sql.Connection connection = - ConnectionUtil.getInputConnection(getUtility().getConfiguration(), new Properties())) { - connection.createStatement().execute(DROP_NONTENANT_VIEW_SQL); - } + dropTenantView(); + dropNonTenantView(); } @Test @@ -139,7 +131,7 @@ public void testSystemCatalogWALEntryFilter() throws Exception { WAL.Entry nonTenantEntryCatalog = getEntry(systemCatalogTableName, nonTenantGetCatalog); WAL.Entry tenantEntryCatalog = getEntry(systemCatalogTableName, tenantGetCatalog); - int tenantRowCount = getAndAssertTenantCountInEdit(tenantEntryCatalog); + int tenantRowCount = getAndAssertCountInEdit(tenantEntryCatalog, true); Assert.assertTrue(tenantRowCount > 0); //verify that the tenant view WAL.Entry passes the filter and the non-tenant view does not @@ -156,7 +148,7 @@ public void testSystemCatalogWALEntryFilter() throws Exception { Assert.assertNotNull("Tenant view was filtered when it shouldn't be!", filteredTenantEntryCatalog); Assert.assertEquals("Not all data for replicated for tenant", tenantRowCount, - getAndAssertTenantCountInEdit(filteredTenantEntryCatalog)); + getAndAssertCountInEdit(filteredTenantEntryCatalog, true)); //now check that a WAL.Entry with cells from both a tenant and a non-tenant //catalog row only allow the tenant cells through @@ -182,7 +174,7 @@ public void testSystemChildLinkWALEntryFilter() throws Exception { WAL.Entry tenantEntryChildLink = getEntry(systemChildLinkTableName, tenantGetChildLink); WAL.Entry nonTenantEntryChildLink = getEntry(systemChildLinkTableName, nonTenantGetChildLink); - int tenantRowCount = getAndAssertTenantCountInEdit(tenantEntryChildLink); + int tenantRowCount = getAndAssertCountInEdit(tenantEntryChildLink, true); Assert.assertTrue(tenantRowCount > 0); //verify that the tenant view WAL.Entry passes the filter and the non-tenant view does not @@ -199,7 +191,7 @@ public void testSystemChildLinkWALEntryFilter() throws Exception { Assert.assertNotNull("Tenant view was filtered when it shouldn't be!", filteredTenantEntryChildLink); Assert.assertEquals("Not all data for replicated for tenant", tenantRowCount, - getAndAssertTenantCountInEdit(filteredTenantEntryChildLink)); + getAndAssertCountInEdit(filteredTenantEntryChildLink, true)); //now check that a WAL.Entry with cells from both a tenant and a non-tenant // child link row only allow the tenant cells through @@ -214,7 +206,49 @@ public void testSystemChildLinkWALEntryFilter() throws Exception { chainWALEntryFilter.filter(comboEntry).getEdit().size()); } - public Get getGet(PTable catalogTable, byte[] tenantId, String viewName) { + /** + * Validates the behavior for parent-child link's delete marker via SystemCatalogWalEntryFilter. + * 1. Filtered for non-tenant views. + * 2. Not filtered for tenant views. + * */ + @Test + public void testDeleteMarkerForParentChildLink() throws Exception{ + // Since for 4.16+ all parent-child links are stored in SYSTEM.CHILD_LINK, only + // checking for that table in this test. + + // Make sure link row exists. + WAL.Entry childLinkEntry = getEntry(systemChildLinkTableName, new Scan(), + false); + int tenantRowCount = getAndAssertCountInEdit(childLinkEntry, true); + int nonTenantRowCount = getAndAssertCountInEdit(childLinkEntry, false); + Assert.assertTrue(tenantRowCount > 0 && nonTenantRowCount > 0 ); + + // Drop both tenant and non-tenant view. + dropTenantView(); + dropNonTenantView(); + + // Delete Marker for non-tenant view should get filtered and for tenant-view it should not. + SystemCatalogWALEntryFilter filter = new SystemCatalogWALEntryFilter(); + // Chain the system catalog WAL entry filter to ChainWALEntryFilter + ChainWALEntryFilter chainWALEntryFilter = new ChainWALEntryFilter(filter); + childLinkEntry = getEntry(systemChildLinkTableName, new Scan(), + false); + int tenantDeleteCountBeforeFilter = getDeleteFamilyCellCountInEntry(childLinkEntry, true); + int nonTenantDeleteCountBeforeFilter = getDeleteFamilyCellCountInEntry(childLinkEntry, false); + // Make sure both tenant and non-tenant delete marker exists before filtering + Assert.assertTrue(tenantDeleteCountBeforeFilter > 0 && nonTenantDeleteCountBeforeFilter > 0 ); + + WAL.Entry filteredEntry = chainWALEntryFilter.filter(childLinkEntry); + int tenantDeleteCountAfterFilter = getDeleteFamilyCellCountInEntry(filteredEntry, true); + int nonTenantDeleteCountAfterFilter = getDeleteFamilyCellCountInEntry(filteredEntry, false); + Assert.assertTrue(tenantDeleteCountAfterFilter == tenantDeleteCountBeforeFilter && nonTenantDeleteCountAfterFilter == 0 ); + + // setup views again. + createTenantView(); + createNonTenantView(); + } + + private Get getGet(PTable catalogTable, byte[] tenantId, String viewName) { byte[][] tenantKeyParts = new byte[5][]; tenantKeyParts[0] = tenantId; tenantKeyParts[1] = Bytes.toBytes(SCHEMA_NAME.toUpperCase()); @@ -228,7 +262,7 @@ public Get getGet(PTable catalogTable, byte[] tenantId, String viewName) { return new Get(key.copyBytes()); } - public Get getGetChildLink(PTable catalogTable, byte[] tenantId, String viewName) { + private Get getGetChildLink(PTable catalogTable, byte[] tenantId, String viewName) { byte[][] tenantKeyParts = new byte[5][]; tenantKeyParts[0] = ByteUtil.EMPTY_BYTE_ARRAY; tenantKeyParts[1] = ByteUtil.EMPTY_BYTE_ARRAY; @@ -249,21 +283,53 @@ private boolean isTenantOwnedCell(Cell cell, String tenantId) { boolean isChildLinkForTenantId = row.contains(tenantId) && CellUtil.matchingQualifier(cell, PhoenixDatabaseMetaData.LINK_TYPE_BYTES); - return isTenantIdLeading || isChildLinkForTenantId; + boolean isDeleteMarkerForLinkRow = row.contains(tenantId) && CellUtil.isDeleteFamily(cell); + return isTenantIdLeading || isChildLinkForTenantId || isDeleteMarkerForLinkRow; } - private int getAndAssertTenantCountInEdit(WAL.Entry entry) { - int count = 0; + /** + * Asserts and returns cell count in the WAL.Entry. if tenantOwned is true, tenant owned cell count is + * returned else non-tenant cell count. + * @Param entry {@link WAL.Entry} + * @Param tenantOwned {@link Boolean} + * */ + private int getAndAssertCountInEdit(WAL.Entry entry, boolean tenantOwned) { + int tenantCount = 0; + int nonTenantCount = 0; for (Cell cell : entry.getEdit().getCells()) { if (isTenantOwnedCell(cell, TENANT_ID)) { - count = count + 1; + tenantCount = tenantCount + 1; + } else { + nonTenantCount = nonTenantCount + 1; } } + int count = tenantOwned ? tenantCount : nonTenantCount; Assert.assertTrue(count > 0); return count; } - public WAL.Entry getEntry(TableName tableName, Get get) throws IOException { + /** + * Returns delete family cell count in the WAL.Entry. if tenantOwned is true, tenant owned cell count is + * returned else non-tenant cell count. + * @Param entry {@link WAL.Entry} + * @Param tenantOwned {@link Boolean} + * */ + private int getDeleteFamilyCellCountInEntry(WAL.Entry entry, boolean tenantOwned) { + int tenantCount = 0; + int nonTenantCount = 0; + for (Cell cell : entry.getEdit().getCells()) { + if (CellUtil.isDeleteFamily(cell)) { + if (isTenantOwnedCell(cell, TENANT_ID)) { + tenantCount = tenantCount + 1; + } else { + nonTenantCount = nonTenantCount + 1; + } + } + } + return tenantOwned ? tenantCount : nonTenantCount; + } + + private WAL.Entry getEntry(TableName tableName, Get get) throws IOException { WAL.Entry entry = null; try(Connection conn = ConnectionFactory.createConnection(getUtility().getConfiguration())){ Table htable = conn.getTable(tableName); @@ -284,4 +350,68 @@ public WAL.Entry getEntry(TableName tableName, Get get) throws IOException { } return entry; } + + private WAL.Entry getEntry(TableName tableName, Scan scan, boolean addIndexedKeyValueCell) + throws IOException { + WAL.Entry entry = null; + try(Connection conn = ConnectionFactory.createConnection(getUtility().getConfiguration())) { + Table htable = conn.getTable(tableName); + scan.setRaw(true); + ResultScanner scanner = htable.getScanner(scan); + WALEdit edit = new WALEdit(); + if (addIndexedKeyValueCell) { + // add IndexedKeyValue type cell as the first cell + edit.add(new IndexedKeyValue()); + } + + for (Result r : scanner) { + if (r != null) { + List cellList = r.listCells(); + for (Cell c : cellList) { + edit.add(c); + } + } + } + Assert.assertFalse("No WALEdits were loaded!", edit.isEmpty()); + WALKeyImpl key = new WALKeyImpl(REGION, tableName, 0, 0, uuid); + entry = new WAL.Entry(key, edit); + } + return entry; + } + + private static void dropTenantView() throws Exception { + Properties tenantProperties = new Properties(); + tenantProperties.setProperty("TenantId", TENANT_ID); + try (java.sql.Connection connection = + ConnectionUtil.getInputConnection(getUtility().getConfiguration(), tenantProperties)) { + connection.createStatement().execute(DROP_TENANT_VIEW_SQL); + connection.commit(); + } + } + + private static void dropNonTenantView() throws Exception { + try (java.sql.Connection connection = + ConnectionUtil.getInputConnection(getUtility().getConfiguration(), new Properties())) { + + connection.createStatement().execute(DROP_NONTENANT_VIEW_SQL); + } + } + + private static void createTenantView() throws Exception { + Properties tenantProperties = new Properties(); + tenantProperties.setProperty("TenantId", TENANT_ID); + try (java.sql.Connection connection = + ConnectionUtil.getInputConnection(getUtility().getConfiguration(), tenantProperties)) { + connection.createStatement().execute(CREATE_TENANT_VIEW_SQL); + connection.commit(); + } + } + + private static void createNonTenantView() throws Exception { + try (java.sql.Connection connection = + ConnectionUtil.getInputConnection(getUtility().getConfiguration(), new Properties())) { + connection.createStatement().execute(CREATE_NONTENANT_VIEW_SQL); + connection.commit(); + } + } } diff --git a/phoenix-core/src/main/java/org/apache/phoenix/replication/SystemCatalogWALEntryFilter.java b/phoenix-core/src/main/java/org/apache/phoenix/replication/SystemCatalogWALEntryFilter.java index 00bfdc84b34..2e2e659a83c 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/replication/SystemCatalogWALEntryFilter.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/replication/SystemCatalogWALEntryFilter.java @@ -92,7 +92,8 @@ private boolean isTenantIdLeadingInKey(final Cell cell) { * tenant id, system.child_link table have tenant owned data for parent child * links. In this case, the column qualifier is * {@code PhoenixDatabaseMetaData#LINK_TYPE_BYTES} and value is - * {@code PTable.LinkType.CHILD_TABLE}. + * {@code PTable.LinkType.CHILD_TABLE}. For corresponding delete markers the + * KeyValue type {@code KeyValue.Type} is {@code KeyValue.Type.DeleteFamily} * @param cell hbase cell * @return true if the cell is tenant owned */ @@ -105,18 +106,18 @@ private boolean isTenantRowCellSystemChildLink(final Cell cell) { if (!isTenantRowCell) { boolean isChildLink = CellUtil.matchingQualifier( cell, PhoenixDatabaseMetaData.LINK_TYPE_BYTES); - if (isChildLink) { - if (CellUtil.matchingValue(cell, CHILD_TABLE_BYTES)) { - byte[][] rowViewKeyMetadata = new byte[NUM_COLUMNS_PRIMARY_KEY][]; - SchemaUtil.getVarChars(key.get(), key.getOffset(), + if ((isChildLink && CellUtil.matchingValue(cell, CHILD_TABLE_BYTES)) || + CellUtil.isDeleteFamily(cell)) { + byte[][] rowViewKeyMetadata = new byte[NUM_COLUMNS_PRIMARY_KEY][]; + SchemaUtil.getVarChars(key.get(), key.getOffset(), key.getLength(), 0, rowViewKeyMetadata); - // if the child link is to a tenant-owned view, - // the COLUMN_NAME field will be the byte[] of the tenant - //otherwise, it will be an empty byte array - // (NOT QueryConstants.SEPARATOR_BYTE, but a byte[0]) - isChildLinkToTenantView = - rowViewKeyMetadata[COLUMN_NAME_INDEX].length != 0; - } + /** if the child link is to a tenant-owned view, the COLUMN_NAME field will be + * the byte[] of the tenant otherwise, it will be an empty byte array + * (NOT QueryConstants.SEPARATOR_BYTE, but a byte[0]). This assumption is also + * true for child link's delete markers in SYSTEM.CHILD_LINK as it only contains link + * rows and does not deal with other type of rows like column rows that also has + * COLUMN_NAME populated with actual column name.**/ + isChildLinkToTenantView = rowViewKeyMetadata[COLUMN_NAME_INDEX].length != 0; } } return isTenantRowCell || isChildLinkToTenantView;