HBASE-22146 SpaceQuotaViolationPolicy Disable is not working in Names…

hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaTableUtil.java

@@ -628,6 +628,34 @@ static List<Delete> createDeletesForExistingSnapshotsFromScan(Connection connect
  }
  }
 
+ /**
+ * Remove table usage snapshots (u:p columns) for the namespace passed
+ * @param connection connection to re-use
+ * @param namespace the namespace to fetch the list of table usage snapshots
+ */
+ static void deleteTableUsageSnapshotsForNamespace(Connection connection, String namespace)
+ throws IOException {
+ Scan s = new Scan();
+ //Get rows for all tables in namespace
+ s.setRowPrefixFilter(Bytes.add(QUOTA_TABLE_ROW_KEY_PREFIX, Bytes.toBytes(namespace + TableName.NAMESPACE_DELIM)));
+ //Scan for table usage column (u:p) in quota table
+ s.addColumn(QUOTA_FAMILY_USAGE,QUOTA_QUALIFIER_POLICY);
+ //Scan for table quota column (q:s) if table has a space quota defined
+ s.addColumn(QUOTA_FAMILY_INFO,QUOTA_QUALIFIER_SETTINGS);
+ try (Table quotaTable = connection.getTable(QUOTA_TABLE_NAME);
+ ResultScanner rs = quotaTable.getScanner(s)) {
+ for (Result r : rs) {
+ byte[] data = r.getValue(QUOTA_FAMILY_INFO, QUOTA_QUALIFIER_SETTINGS);
+ //if table does not have a table space quota defined, delete table usage column (u:p)
+ if (data == null) {
+ Delete delete = new Delete(r.getRow());
+ delete.addColumns(QUOTA_FAMILY_USAGE,QUOTA_QUALIFIER_POLICY);
+ quotaTable.delete(delete);
+ }
+ }
+ }
+ }
+
  /**
  * Fetches the computed size of all snapshots against tables in a namespace for space quotas.
  */

hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaUtil.java

@@ -266,6 +266,14 @@ private static void deleteQuotas(final Connection connection, final byte[] rowKe
  if (qualifier != null) {
  delete.addColumns(QUOTA_FAMILY_INFO, qualifier);
  }
+ if (isNamespaceRowKey(rowKey)) {
+ String ns = getNamespaceFromRowKey(rowKey);
+ Quotas namespaceQuota = getNamespaceQuota(connection,ns);
+ if (namespaceQuota != null && namespaceQuota.hasSpace()) {
+ // When deleting namespace space quota, also delete table usage(u:p) snapshots
+ deleteTableUsageSnapshotsForNamespace(connection, ns);
+ }
+ }
  doDelete(connection, delete);
  }
 

hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/SpaceQuotaHelperForTests.java

@@ -158,6 +158,28 @@ TableName writeUntilViolation(SpaceViolationPolicy policyToViolate) throws Excep
  return tn;
  }
 
+
+ TableName writeUntilViolationAndVerifyViolationInNamespace(
+ String ns, SpaceViolationPolicy policyToViolate, Mutation m) throws Exception {
+ final TableName tn = writeUntilViolationInNamespace(ns, policyToViolate);
+ verifyViolation(policyToViolate, tn, m);
+ return tn;
+ }
+
+ TableName writeUntilViolationInNamespace(String ns, SpaceViolationPolicy policyToViolate) throws Exception {
+ TableName tn = createTableWithRegions(ns,10);
+
+ setQuotaLimit(ns, policyToViolate, 4L);
+
+ // Write more data than should be allowed and flush it to disk
+ writeData(tn, 5L * SpaceQuotaHelperForTests.ONE_MEGABYTE);
+
+ // This should be sufficient time for the chores to run and see the change.
+ Thread.sleep(5000);
+
+ return tn;
+ }
+
  /**
  * Verifies that the given policy on the given table has been violated
  */
@@ -260,6 +282,19 @@ void verifyNoViolation(TableName tn, Mutation m) throws Exception {
  assertTrue("Expected to succeed in writing data to a table not having quota ", sawSuccess);
  }
 
+ /**
+ * Verifies that table usage snapshot exists for the table
+ */
+ void verifyTableUsageSnapshotForSpaceQuotaExist(TableName tn) throws Exception {
+ boolean sawUsageSnapshot = false;
+ try (Table quotaTable = testUtil.getConnection().getTable(QuotaTableUtil.QUOTA_TABLE_NAME)) {
+ Scan s = QuotaTableUtil.makeQuotaSnapshotScanForTable(tn);
+ ResultScanner rs = quotaTable.getScanner(s);
+ sawUsageSnapshot = (rs.next() != null);
+ }
+ assertTrue("Expected to succeed in getting table usage snapshots for space quota", sawUsageSnapshot);
+ }
+
  /**
  * Sets the given quota (policy & limit) on the passed table.
  */
@@ -271,6 +306,17 @@ void setQuotaLimit(final TableName tn, SpaceViolationPolicy policy, long sizeInM
  LOG.debug("Quota limit set for table = {}, limit = {}", tn, sizeLimit);
  }
 
+ /**
+ * Sets the given quota (policy & limit) on the passed namespace.
+ */
+ void setQuotaLimit(String ns, SpaceViolationPolicy policy, long sizeInMBs)
+ throws Exception {
+ final long sizeLimit = sizeInMBs * SpaceQuotaHelperForTests.ONE_MEGABYTE;
+ QuotaSettings settings = QuotaSettingsFactory.limitNamespaceSpace(ns, sizeLimit, policy);
+ testUtil.getAdmin().setQuota(settings);
+ LOG.debug("Quota limit set for namespace = {}, limit = {}", ns, sizeLimit);
+ }
+
  /**
  * Removes the space quota from the given table
  */
@@ -280,6 +326,16 @@ void removeQuotaFromtable(final TableName tn) throws Exception {
  LOG.debug("Space quota settings removed from the table ", tn);
  }
 
+ /**
+ * Removes the space quota from the given namespace
+ */
+ void removeQuotaFromNamespace(String ns) throws Exception {
+ QuotaSettings removeQuota = QuotaSettingsFactory.removeNamespaceSpaceLimit(ns);
+ Admin admin = testUtil.getAdmin();
+ admin.setQuota(removeQuota);
+ LOG.debug("Space quota settings removed from the namespace ", ns);
+ }
+
  /**
  * Removes all quotas defined in the HBase quota table.
  */
@@ -411,7 +467,13 @@ void writeData(TableName tn, long sizeInBytes, byte[] qual) throws IOException {
  }
 
  NamespaceDescriptor createNamespace() throws Exception {
- NamespaceDescriptor nd = NamespaceDescriptor.create("ns" + counter.getAndIncrement()).build();
+ return createNamespace(null);
+ }
+
+ NamespaceDescriptor createNamespace(String namespace) throws Exception {
+ if (namespace == null || namespace.trim().isEmpty())
+ namespace = "ns" + counter.getAndIncrement();
+ NamespaceDescriptor nd = NamespaceDescriptor.create(namespace).build();
  testUtil.getAdmin().createNamespace(nd);
  return nd;
  }

hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestSpaceQuotaRemoval.java

@@ -20,6 +20,7 @@
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HBaseClassTestRule;
 import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.NamespaceDescriptor;
 import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.testclassification.LargeTests;
@@ -139,6 +140,63 @@ private void setQuotaAndThenRemove(SpaceViolationPolicy policy) throws Exception
  helper.verifyNoViolation(tn, put);
  }
 
+ @Test
+ public void testDeleteTableUsageSnapshotsForNamespace() throws Exception {
+ Put put = new Put(Bytes.toBytes("to_reject"));
+ put.addColumn(Bytes.toBytes(SpaceQuotaHelperForTests.F1), Bytes.toBytes("to"),
+ Bytes.toBytes("reject"));
+
+ SpaceViolationPolicy policy = SpaceViolationPolicy.NO_INSERTS;
+
+ //Create a namespace
+ String ns1 = "nsnew";
+ NamespaceDescriptor nsd = helper.createNamespace(ns1);
+
+ //Create 2nd namespace with name similar to ns1
+ String ns2 = ns1 + "test";
+ NamespaceDescriptor nsd2 = helper.createNamespace(ns2);
+
+ // Do puts until we violate space policy on table tn1 in namesapce ns1
+ final TableName tn1 = helper.writeUntilViolationAndVerifyViolationInNamespace(ns1, policy, put);
+
+ // Do puts until we violate space policy on table tn2 in namespace ns2
+ final TableName tn2 = helper.writeUntilViolationAndVerifyViolationInNamespace(ns2, policy, put);
+
+ // Now, remove the quota from namespace ns1 which will remove table usage snapshots for ns1
+ helper.removeQuotaFromNamespace(ns1);
+
+ // Verify that table usage snapshot for table tn2 in namespace ns2 exist
+ helper.verifyTableUsageSnapshotForSpaceQuotaExist(tn2);
+
+ // Put a new row on tn2: should violate as space quota exists on namespace ns2
+ helper.verifyViolation(policy, tn2, put);
+
+ // Put a new row on tn1: should not violate as quota settings removed from namespace ns1
+ helper.verifyNoViolation(tn1, put);
+ }
+
+ @Test
+ public void testSetNamespaceSizeQuotaAndThenRemove() throws Exception {
+ Put put = new Put(Bytes.toBytes("to_reject"));
+ put.addColumn(Bytes.toBytes(SpaceQuotaHelperForTests.F1), Bytes.toBytes("to"),
+ Bytes.toBytes("reject"));
+
+ SpaceViolationPolicy policy = SpaceViolationPolicy.NO_INSERTS;
+
+ //Create namespace
+ NamespaceDescriptor nsd = helper.createNamespace();
+ String ns = nsd.getName();
+
+ // Do puts until we violate space policy on table tn1
+ final TableName tn1 = helper.writeUntilViolationAndVerifyViolationInNamespace(ns, policy, put);
+
+ // Now, remove the quota from namespace
+ helper.removeQuotaFromNamespace(ns);
+
+ // Put a new row now on tn1: should not violate as quota settings removed from namespace
+ helper.verifyNoViolation(tn1, put);
+ }
+
  private void setQuotaAndThenRemoveInOneAmongTwoTables(SpaceViolationPolicy policy)
  throws Exception {
  Put put = new Put(Bytes.toBytes("to_reject"));