From 5ebcd9e2f100ab7eb15452e41ae27be8cbe4070e Mon Sep 17 00:00:00 2001 From: Ankit Singhal Date: Fri, 15 Jan 2016 16:00:23 +0530 Subject: [PATCH 01/47] PHOENIX-2417 Compress memory used by row key byte[] of guideposts --- .../apache/phoenix/compile/ScanRanges.java | 10 +- .../UngroupedAggregateRegionObserver.java | 1 + .../generated/PGuidePostsProtos.java | 370 +++-- .../coprocessor/generated/PTableProtos.java | 349 ++--- .../generated/StatCollectorProtos.java | 1269 ----------------- .../org/apache/phoenix/execute/ScanPlan.java | 2 +- .../phoenix/iterate/BaseResultIterators.java | 144 +- .../org/apache/phoenix/schema/PTableImpl.java | 32 +- .../phoenix/schema/stats/GuidePostsInfo.java | 149 +- .../phoenix/schema/stats/PTableStatsImpl.java | 57 +- .../schema/stats/StatisticsCollector.java | 33 +- .../schema/stats/StatisticsScanner.java | 4 +- .../phoenix/schema/stats/StatisticsUtil.java | 28 +- .../schema/stats/StatisticsWriter.java | 49 +- .../org/apache/phoenix/util/ByteUtil.java | 2 + .../org/apache/phoenix/util/CodecUtils.java | 89 ++ .../phoenix/util/PrefixByteDecoder.java | 85 ++ .../phoenix/util/PrefixByteEncoder.java | 99 ++ .../phoenix/filter/SkipScanBigFilterTest.java | 10 +- .../util/PrefixByteEncoderDecoderTest.java | 96 ++ phoenix-protocol/src/main/PGuidePosts.proto | 4 +- phoenix-protocol/src/main/PTable.proto | 3 +- 22 files changed, 1130 insertions(+), 1755 deletions(-) delete mode 100644 phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/StatCollectorProtos.java create mode 100644 phoenix-core/src/main/java/org/apache/phoenix/util/CodecUtils.java create mode 100644 phoenix-core/src/main/java/org/apache/phoenix/util/PrefixByteDecoder.java create mode 100644 phoenix-core/src/main/java/org/apache/phoenix/util/PrefixByteEncoder.java create mode 100644 phoenix-core/src/test/java/org/apache/phoenix/util/PrefixByteEncoderDecoderTest.java diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/ScanRanges.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/ScanRanges.java index 4d343f342e8..91d07ac898d 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/compile/ScanRanges.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/ScanRanges.java @@ -237,7 +237,9 @@ private static byte[] stripPrefix(byte[] key, int keyOffset) { return temp; } - public Scan intersectScan(Scan scan, final byte[] originalStartKey, final byte[] originalStopKey, final int keyOffset, boolean crossesRegionBoundary) { + public Scan intersectScan(Scan scan, final ImmutableBytesWritable originalStartKeyPtr, final ImmutableBytesWritable originalStopKeyPtr, final int keyOffset, boolean crossesRegionBoundary) { + byte[] originalStartKey= originalStartKeyPtr.get(); + byte[] originalStopKey= originalStopKeyPtr.get(); byte[] startKey = originalStartKey; byte[] stopKey = originalStopKey; if (stopKey.length > 0 && Bytes.compareTo(startKey, stopKey) >= 0) { @@ -384,8 +386,8 @@ public Scan intersectScan(Scan scan, final byte[] originalStartKey, final byte[] if (scanStopKey.length > 0 && Bytes.compareTo(scanStartKey, scanStopKey) >= 0) { return null; } - newScan.setStartRow(scanStartKey); - newScan.setStopRow(scanStopKey); + newScan.setStartRow(Bytes.copy(scanStartKey)); + newScan.setStopRow(Bytes.copy(scanStopKey)); if(keyOffset > 0) { newScan.setAttribute(STARTKEY_OFFSET, Bytes.toBytes(keyOffset)); } @@ -416,7 +418,7 @@ public boolean intersects(byte[] lowerInclusiveKey, byte[] upperExclusiveKey, in } //return filter.hasIntersect(lowerInclusiveKey, upperExclusiveKey); - return intersectScan(null, lowerInclusiveKey, upperExclusiveKey, keyOffset, crossesRegionBoundary) == HAS_INTERSECTION; + return intersectScan(null, new ImmutableBytesWritable(lowerInclusiveKey), new ImmutableBytesWritable(upperExclusiveKey), keyOffset, crossesRegionBoundary) == HAS_INTERSECTION; } public SkipScanFilter getSkipScanFilter() { diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java index 1d67994747e..7fb9ff65c74 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java @@ -783,6 +783,7 @@ private long collectStatsInternal() throws IOException { } finally { try { if (noErrors && !compactionRunning) { + stats.updateGuidePosts(); stats.updateStatistic(region); logger.info("UPDATE STATISTICS finished successfully for scanner: " + innerScanner + ". Number of rows scanned: " + rowCount diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/PGuidePostsProtos.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/PGuidePostsProtos.java index 9f965309b0e..58999740ed8 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/PGuidePostsProtos.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/PGuidePostsProtos.java @@ -11,19 +11,15 @@ public static void registerAllExtensions( public interface PGuidePostsOrBuilder extends com.google.protobuf.MessageOrBuilder { - // repeated bytes guidePosts = 1; + // optional bytes guidePosts = 1; /** - * repeated bytes guidePosts = 1; + * optional bytes guidePosts = 1; */ - java.util.List getGuidePostsList(); + boolean hasGuidePosts(); /** - * repeated bytes guidePosts = 1; + * optional bytes guidePosts = 1; */ - int getGuidePostsCount(); - /** - * repeated bytes guidePosts = 1; - */ - com.google.protobuf.ByteString getGuidePosts(int index); + com.google.protobuf.ByteString getGuidePosts(); // optional int64 byteCount = 2; /** @@ -44,6 +40,26 @@ public interface PGuidePostsOrBuilder * optional int64 rowCount = 3; */ long getRowCount(); + + // optional int32 maxLength = 4; + /** + * optional int32 maxLength = 4; + */ + boolean hasMaxLength(); + /** + * optional int32 maxLength = 4; + */ + int getMaxLength(); + + // optional int32 guidePostsCount = 5; + /** + * optional int32 guidePostsCount = 5; + */ + boolean hasGuidePostsCount(); + /** + * optional int32 guidePostsCount = 5; + */ + int getGuidePostsCount(); } /** * Protobuf type {@code PGuidePosts} @@ -97,23 +113,30 @@ private PGuidePosts( break; } case 10: { - if (!((mutable_bitField0_ & 0x00000001) == 0x00000001)) { - guidePosts_ = new java.util.ArrayList(); - mutable_bitField0_ |= 0x00000001; - } - guidePosts_.add(input.readBytes()); + bitField0_ |= 0x00000001; + guidePosts_ = input.readBytes(); break; } case 16: { - bitField0_ |= 0x00000001; + bitField0_ |= 0x00000002; byteCount_ = input.readInt64(); break; } case 24: { - bitField0_ |= 0x00000002; + bitField0_ |= 0x00000004; rowCount_ = input.readInt64(); break; } + case 32: { + bitField0_ |= 0x00000008; + maxLength_ = input.readInt32(); + break; + } + case 40: { + bitField0_ |= 0x00000010; + guidePostsCount_ = input.readInt32(); + break; + } } } } catch (com.google.protobuf.InvalidProtocolBufferException e) { @@ -122,9 +145,6 @@ private PGuidePosts( throw new com.google.protobuf.InvalidProtocolBufferException( e.getMessage()).setUnfinishedMessage(this); } finally { - if (((mutable_bitField0_ & 0x00000001) == 0x00000001)) { - guidePosts_ = java.util.Collections.unmodifiableList(guidePosts_); - } this.unknownFields = unknownFields.build(); makeExtensionsImmutable(); } @@ -157,27 +177,20 @@ public com.google.protobuf.Parser getParserForType() { } private int bitField0_; - // repeated bytes guidePosts = 1; + // optional bytes guidePosts = 1; public static final int GUIDEPOSTS_FIELD_NUMBER = 1; - private java.util.List guidePosts_; - /** - * repeated bytes guidePosts = 1; - */ - public java.util.List - getGuidePostsList() { - return guidePosts_; - } + private com.google.protobuf.ByteString guidePosts_; /** - * repeated bytes guidePosts = 1; + * optional bytes guidePosts = 1; */ - public int getGuidePostsCount() { - return guidePosts_.size(); + public boolean hasGuidePosts() { + return ((bitField0_ & 0x00000001) == 0x00000001); } /** - * repeated bytes guidePosts = 1; + * optional bytes guidePosts = 1; */ - public com.google.protobuf.ByteString getGuidePosts(int index) { - return guidePosts_.get(index); + public com.google.protobuf.ByteString getGuidePosts() { + return guidePosts_; } // optional int64 byteCount = 2; @@ -187,7 +200,7 @@ public com.google.protobuf.ByteString getGuidePosts(int index) { * optional int64 byteCount = 2; */ public boolean hasByteCount() { - return ((bitField0_ & 0x00000001) == 0x00000001); + return ((bitField0_ & 0x00000002) == 0x00000002); } /** * optional int64 byteCount = 2; @@ -203,7 +216,7 @@ public long getByteCount() { * optional int64 rowCount = 3; */ public boolean hasRowCount() { - return ((bitField0_ & 0x00000002) == 0x00000002); + return ((bitField0_ & 0x00000004) == 0x00000004); } /** * optional int64 rowCount = 3; @@ -212,10 +225,44 @@ public long getRowCount() { return rowCount_; } + // optional int32 maxLength = 4; + public static final int MAXLENGTH_FIELD_NUMBER = 4; + private int maxLength_; + /** + * optional int32 maxLength = 4; + */ + public boolean hasMaxLength() { + return ((bitField0_ & 0x00000008) == 0x00000008); + } + /** + * optional int32 maxLength = 4; + */ + public int getMaxLength() { + return maxLength_; + } + + // optional int32 guidePostsCount = 5; + public static final int GUIDEPOSTSCOUNT_FIELD_NUMBER = 5; + private int guidePostsCount_; + /** + * optional int32 guidePostsCount = 5; + */ + public boolean hasGuidePostsCount() { + return ((bitField0_ & 0x00000010) == 0x00000010); + } + /** + * optional int32 guidePostsCount = 5; + */ + public int getGuidePostsCount() { + return guidePostsCount_; + } + private void initFields() { - guidePosts_ = java.util.Collections.emptyList(); + guidePosts_ = com.google.protobuf.ByteString.EMPTY; byteCount_ = 0L; rowCount_ = 0L; + maxLength_ = 0; + guidePostsCount_ = 0; } private byte memoizedIsInitialized = -1; public final boolean isInitialized() { @@ -229,15 +276,21 @@ public final boolean isInitialized() { public void writeTo(com.google.protobuf.CodedOutputStream output) throws java.io.IOException { getSerializedSize(); - for (int i = 0; i < guidePosts_.size(); i++) { - output.writeBytes(1, guidePosts_.get(i)); - } if (((bitField0_ & 0x00000001) == 0x00000001)) { - output.writeInt64(2, byteCount_); + output.writeBytes(1, guidePosts_); } if (((bitField0_ & 0x00000002) == 0x00000002)) { + output.writeInt64(2, byteCount_); + } + if (((bitField0_ & 0x00000004) == 0x00000004)) { output.writeInt64(3, rowCount_); } + if (((bitField0_ & 0x00000008) == 0x00000008)) { + output.writeInt32(4, maxLength_); + } + if (((bitField0_ & 0x00000010) == 0x00000010)) { + output.writeInt32(5, guidePostsCount_); + } getUnknownFields().writeTo(output); } @@ -247,23 +300,26 @@ public int getSerializedSize() { if (size != -1) return size; size = 0; - { - int dataSize = 0; - for (int i = 0; i < guidePosts_.size(); i++) { - dataSize += com.google.protobuf.CodedOutputStream - .computeBytesSizeNoTag(guidePosts_.get(i)); - } - size += dataSize; - size += 1 * getGuidePostsList().size(); - } if (((bitField0_ & 0x00000001) == 0x00000001)) { size += com.google.protobuf.CodedOutputStream - .computeInt64Size(2, byteCount_); + .computeBytesSize(1, guidePosts_); } if (((bitField0_ & 0x00000002) == 0x00000002)) { + size += com.google.protobuf.CodedOutputStream + .computeInt64Size(2, byteCount_); + } + if (((bitField0_ & 0x00000004) == 0x00000004)) { size += com.google.protobuf.CodedOutputStream .computeInt64Size(3, rowCount_); } + if (((bitField0_ & 0x00000008) == 0x00000008)) { + size += com.google.protobuf.CodedOutputStream + .computeInt32Size(4, maxLength_); + } + if (((bitField0_ & 0x00000010) == 0x00000010)) { + size += com.google.protobuf.CodedOutputStream + .computeInt32Size(5, guidePostsCount_); + } size += getUnknownFields().getSerializedSize(); memoizedSerializedSize = size; return size; @@ -287,8 +343,11 @@ public boolean equals(final java.lang.Object obj) { org.apache.phoenix.coprocessor.generated.PGuidePostsProtos.PGuidePosts other = (org.apache.phoenix.coprocessor.generated.PGuidePostsProtos.PGuidePosts) obj; boolean result = true; - result = result && getGuidePostsList() - .equals(other.getGuidePostsList()); + result = result && (hasGuidePosts() == other.hasGuidePosts()); + if (hasGuidePosts()) { + result = result && getGuidePosts() + .equals(other.getGuidePosts()); + } result = result && (hasByteCount() == other.hasByteCount()); if (hasByteCount()) { result = result && (getByteCount() @@ -299,6 +358,16 @@ public boolean equals(final java.lang.Object obj) { result = result && (getRowCount() == other.getRowCount()); } + result = result && (hasMaxLength() == other.hasMaxLength()); + if (hasMaxLength()) { + result = result && (getMaxLength() + == other.getMaxLength()); + } + result = result && (hasGuidePostsCount() == other.hasGuidePostsCount()); + if (hasGuidePostsCount()) { + result = result && (getGuidePostsCount() + == other.getGuidePostsCount()); + } result = result && getUnknownFields().equals(other.getUnknownFields()); return result; @@ -312,9 +381,9 @@ public int hashCode() { } int hash = 41; hash = (19 * hash) + getDescriptorForType().hashCode(); - if (getGuidePostsCount() > 0) { + if (hasGuidePosts()) { hash = (37 * hash) + GUIDEPOSTS_FIELD_NUMBER; - hash = (53 * hash) + getGuidePostsList().hashCode(); + hash = (53 * hash) + getGuidePosts().hashCode(); } if (hasByteCount()) { hash = (37 * hash) + BYTECOUNT_FIELD_NUMBER; @@ -324,6 +393,14 @@ public int hashCode() { hash = (37 * hash) + ROWCOUNT_FIELD_NUMBER; hash = (53 * hash) + hashLong(getRowCount()); } + if (hasMaxLength()) { + hash = (37 * hash) + MAXLENGTH_FIELD_NUMBER; + hash = (53 * hash) + getMaxLength(); + } + if (hasGuidePostsCount()) { + hash = (37 * hash) + GUIDEPOSTSCOUNT_FIELD_NUMBER; + hash = (53 * hash) + getGuidePostsCount(); + } hash = (29 * hash) + getUnknownFields().hashCode(); memoizedHashCode = hash; return hash; @@ -433,12 +510,16 @@ private static Builder create() { public Builder clear() { super.clear(); - guidePosts_ = java.util.Collections.emptyList(); + guidePosts_ = com.google.protobuf.ByteString.EMPTY; bitField0_ = (bitField0_ & ~0x00000001); byteCount_ = 0L; bitField0_ = (bitField0_ & ~0x00000002); rowCount_ = 0L; bitField0_ = (bitField0_ & ~0x00000004); + maxLength_ = 0; + bitField0_ = (bitField0_ & ~0x00000008); + guidePostsCount_ = 0; + bitField0_ = (bitField0_ & ~0x00000010); return this; } @@ -467,19 +548,26 @@ public org.apache.phoenix.coprocessor.generated.PGuidePostsProtos.PGuidePosts bu org.apache.phoenix.coprocessor.generated.PGuidePostsProtos.PGuidePosts result = new org.apache.phoenix.coprocessor.generated.PGuidePostsProtos.PGuidePosts(this); int from_bitField0_ = bitField0_; int to_bitField0_ = 0; - if (((bitField0_ & 0x00000001) == 0x00000001)) { - guidePosts_ = java.util.Collections.unmodifiableList(guidePosts_); - bitField0_ = (bitField0_ & ~0x00000001); + if (((from_bitField0_ & 0x00000001) == 0x00000001)) { + to_bitField0_ |= 0x00000001; } result.guidePosts_ = guidePosts_; if (((from_bitField0_ & 0x00000002) == 0x00000002)) { - to_bitField0_ |= 0x00000001; + to_bitField0_ |= 0x00000002; } result.byteCount_ = byteCount_; if (((from_bitField0_ & 0x00000004) == 0x00000004)) { - to_bitField0_ |= 0x00000002; + to_bitField0_ |= 0x00000004; } result.rowCount_ = rowCount_; + if (((from_bitField0_ & 0x00000008) == 0x00000008)) { + to_bitField0_ |= 0x00000008; + } + result.maxLength_ = maxLength_; + if (((from_bitField0_ & 0x00000010) == 0x00000010)) { + to_bitField0_ |= 0x00000010; + } + result.guidePostsCount_ = guidePostsCount_; result.bitField0_ = to_bitField0_; onBuilt(); return result; @@ -496,15 +584,8 @@ public Builder mergeFrom(com.google.protobuf.Message other) { public Builder mergeFrom(org.apache.phoenix.coprocessor.generated.PGuidePostsProtos.PGuidePosts other) { if (other == org.apache.phoenix.coprocessor.generated.PGuidePostsProtos.PGuidePosts.getDefaultInstance()) return this; - if (!other.guidePosts_.isEmpty()) { - if (guidePosts_.isEmpty()) { - guidePosts_ = other.guidePosts_; - bitField0_ = (bitField0_ & ~0x00000001); - } else { - ensureGuidePostsIsMutable(); - guidePosts_.addAll(other.guidePosts_); - } - onChanged(); + if (other.hasGuidePosts()) { + setGuidePosts(other.getGuidePosts()); } if (other.hasByteCount()) { setByteCount(other.getByteCount()); @@ -512,6 +593,12 @@ public Builder mergeFrom(org.apache.phoenix.coprocessor.generated.PGuidePostsPro if (other.hasRowCount()) { setRowCount(other.getRowCount()); } + if (other.hasMaxLength()) { + setMaxLength(other.getMaxLength()); + } + if (other.hasGuidePostsCount()) { + setGuidePostsCount(other.getGuidePostsCount()); + } this.mergeUnknownFields(other.getUnknownFields()); return this; } @@ -539,74 +626,38 @@ public Builder mergeFrom( } private int bitField0_; - // repeated bytes guidePosts = 1; - private java.util.List guidePosts_ = java.util.Collections.emptyList(); - private void ensureGuidePostsIsMutable() { - if (!((bitField0_ & 0x00000001) == 0x00000001)) { - guidePosts_ = new java.util.ArrayList(guidePosts_); - bitField0_ |= 0x00000001; - } - } + // optional bytes guidePosts = 1; + private com.google.protobuf.ByteString guidePosts_ = com.google.protobuf.ByteString.EMPTY; /** - * repeated bytes guidePosts = 1; + * optional bytes guidePosts = 1; */ - public java.util.List - getGuidePostsList() { - return java.util.Collections.unmodifiableList(guidePosts_); + public boolean hasGuidePosts() { + return ((bitField0_ & 0x00000001) == 0x00000001); } /** - * repeated bytes guidePosts = 1; + * optional bytes guidePosts = 1; */ - public int getGuidePostsCount() { - return guidePosts_.size(); + public com.google.protobuf.ByteString getGuidePosts() { + return guidePosts_; } /** - * repeated bytes guidePosts = 1; + * optional bytes guidePosts = 1; */ - public com.google.protobuf.ByteString getGuidePosts(int index) { - return guidePosts_.get(index); - } - /** - * repeated bytes guidePosts = 1; - */ - public Builder setGuidePosts( - int index, com.google.protobuf.ByteString value) { - if (value == null) { - throw new NullPointerException(); - } - ensureGuidePostsIsMutable(); - guidePosts_.set(index, value); - onChanged(); - return this; - } - /** - * repeated bytes guidePosts = 1; - */ - public Builder addGuidePosts(com.google.protobuf.ByteString value) { + public Builder setGuidePosts(com.google.protobuf.ByteString value) { if (value == null) { throw new NullPointerException(); } - ensureGuidePostsIsMutable(); - guidePosts_.add(value); - onChanged(); - return this; - } - /** - * repeated bytes guidePosts = 1; - */ - public Builder addAllGuidePosts( - java.lang.Iterable values) { - ensureGuidePostsIsMutable(); - super.addAll(values, guidePosts_); + bitField0_ |= 0x00000001; + guidePosts_ = value; onChanged(); return this; } /** - * repeated bytes guidePosts = 1; + * optional bytes guidePosts = 1; */ public Builder clearGuidePosts() { - guidePosts_ = java.util.Collections.emptyList(); bitField0_ = (bitField0_ & ~0x00000001); + guidePosts_ = getDefaultInstance().getGuidePosts(); onChanged(); return this; } @@ -677,6 +728,72 @@ public Builder clearRowCount() { return this; } + // optional int32 maxLength = 4; + private int maxLength_ ; + /** + * optional int32 maxLength = 4; + */ + public boolean hasMaxLength() { + return ((bitField0_ & 0x00000008) == 0x00000008); + } + /** + * optional int32 maxLength = 4; + */ + public int getMaxLength() { + return maxLength_; + } + /** + * optional int32 maxLength = 4; + */ + public Builder setMaxLength(int value) { + bitField0_ |= 0x00000008; + maxLength_ = value; + onChanged(); + return this; + } + /** + * optional int32 maxLength = 4; + */ + public Builder clearMaxLength() { + bitField0_ = (bitField0_ & ~0x00000008); + maxLength_ = 0; + onChanged(); + return this; + } + + // optional int32 guidePostsCount = 5; + private int guidePostsCount_ ; + /** + * optional int32 guidePostsCount = 5; + */ + public boolean hasGuidePostsCount() { + return ((bitField0_ & 0x00000010) == 0x00000010); + } + /** + * optional int32 guidePostsCount = 5; + */ + public int getGuidePostsCount() { + return guidePostsCount_; + } + /** + * optional int32 guidePostsCount = 5; + */ + public Builder setGuidePostsCount(int value) { + bitField0_ |= 0x00000010; + guidePostsCount_ = value; + onChanged(); + return this; + } + /** + * optional int32 guidePostsCount = 5; + */ + public Builder clearGuidePostsCount() { + bitField0_ = (bitField0_ & ~0x00000010); + guidePostsCount_ = 0; + onChanged(); + return this; + } + // @@protoc_insertion_point(builder_scope:PGuidePosts) } @@ -702,11 +819,12 @@ public Builder clearRowCount() { descriptor; static { java.lang.String[] descriptorData = { - "\n\021PGuidePosts.proto\"F\n\013PGuidePosts\022\022\n\ngu" + - "idePosts\030\001 \003(\014\022\021\n\tbyteCount\030\002 \001(\003\022\020\n\010row" + - "Count\030\003 \001(\003BE\n(org.apache.phoenix.coproc" + - "essor.generatedB\021PGuidePostsProtosH\001\210\001\001\240" + - "\001\001" + "\n\021PGuidePosts.proto\"r\n\013PGuidePosts\022\022\n\ngu" + + "idePosts\030\001 \001(\014\022\021\n\tbyteCount\030\002 \001(\003\022\020\n\010row" + + "Count\030\003 \001(\003\022\021\n\tmaxLength\030\004 \001(\005\022\027\n\017guideP" + + "ostsCount\030\005 \001(\005BE\n(org.apache.phoenix.co" + + "processor.generatedB\021PGuidePostsProtosH\001" + + "\210\001\001\240\001\001" }; com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner = new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() { @@ -718,7 +836,7 @@ public com.google.protobuf.ExtensionRegistry assignDescriptors( internal_static_PGuidePosts_fieldAccessorTable = new com.google.protobuf.GeneratedMessage.FieldAccessorTable( internal_static_PGuidePosts_descriptor, - new java.lang.String[] { "GuidePosts", "ByteCount", "RowCount", }); + new java.lang.String[] { "GuidePosts", "ByteCount", "RowCount", "MaxLength", "GuidePostsCount", }); return null; } }; diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/PTableProtos.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/PTableProtos.java index 7e71cd9fa8a..1c5c0388f1b 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/PTableProtos.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/PTableProtos.java @@ -1843,19 +1843,15 @@ public interface PTableStatsOrBuilder */ com.google.protobuf.ByteString getKey(); - // repeated bytes values = 2; + // optional bytes guidePosts = 2; /** - * repeated bytes values = 2; + * optional bytes guidePosts = 2; */ - java.util.List getValuesList(); + boolean hasGuidePosts(); /** - * repeated bytes values = 2; + * optional bytes guidePosts = 2; */ - int getValuesCount(); - /** - * repeated bytes values = 2; - */ - com.google.protobuf.ByteString getValues(int index); + com.google.protobuf.ByteString getGuidePosts(); // optional int64 guidePostsByteCount = 3; /** @@ -1900,6 +1896,16 @@ public interface PTableStatsOrBuilder * optional .PGuidePosts pGuidePosts = 6; */ org.apache.phoenix.coprocessor.generated.PGuidePostsProtos.PGuidePostsOrBuilder getPGuidePostsOrBuilder(); + + // optional int32 maxLength = 7; + /** + * optional int32 maxLength = 7; + */ + boolean hasMaxLength(); + /** + * optional int32 maxLength = 7; + */ + int getMaxLength(); } /** * Protobuf type {@code PTableStats} @@ -1958,31 +1964,28 @@ private PTableStats( break; } case 18: { - if (!((mutable_bitField0_ & 0x00000002) == 0x00000002)) { - values_ = new java.util.ArrayList(); - mutable_bitField0_ |= 0x00000002; - } - values_.add(input.readBytes()); + bitField0_ |= 0x00000002; + guidePosts_ = input.readBytes(); break; } case 24: { - bitField0_ |= 0x00000002; + bitField0_ |= 0x00000004; guidePostsByteCount_ = input.readInt64(); break; } case 32: { - bitField0_ |= 0x00000004; + bitField0_ |= 0x00000008; keyBytesCount_ = input.readInt64(); break; } case 40: { - bitField0_ |= 0x00000008; + bitField0_ |= 0x00000010; guidePostsCount_ = input.readInt32(); break; } case 50: { org.apache.phoenix.coprocessor.generated.PGuidePostsProtos.PGuidePosts.Builder subBuilder = null; - if (((bitField0_ & 0x00000010) == 0x00000010)) { + if (((bitField0_ & 0x00000020) == 0x00000020)) { subBuilder = pGuidePosts_.toBuilder(); } pGuidePosts_ = input.readMessage(org.apache.phoenix.coprocessor.generated.PGuidePostsProtos.PGuidePosts.PARSER, extensionRegistry); @@ -1990,7 +1993,12 @@ private PTableStats( subBuilder.mergeFrom(pGuidePosts_); pGuidePosts_ = subBuilder.buildPartial(); } - bitField0_ |= 0x00000010; + bitField0_ |= 0x00000020; + break; + } + case 56: { + bitField0_ |= 0x00000040; + maxLength_ = input.readInt32(); break; } } @@ -2001,9 +2009,6 @@ private PTableStats( throw new com.google.protobuf.InvalidProtocolBufferException( e.getMessage()).setUnfinishedMessage(this); } finally { - if (((mutable_bitField0_ & 0x00000002) == 0x00000002)) { - values_ = java.util.Collections.unmodifiableList(values_); - } this.unknownFields = unknownFields.build(); makeExtensionsImmutable(); } @@ -2052,27 +2057,20 @@ public com.google.protobuf.ByteString getKey() { return key_; } - // repeated bytes values = 2; - public static final int VALUES_FIELD_NUMBER = 2; - private java.util.List values_; + // optional bytes guidePosts = 2; + public static final int GUIDEPOSTS_FIELD_NUMBER = 2; + private com.google.protobuf.ByteString guidePosts_; /** - * repeated bytes values = 2; + * optional bytes guidePosts = 2; */ - public java.util.List - getValuesList() { - return values_; - } - /** - * repeated bytes values = 2; - */ - public int getValuesCount() { - return values_.size(); + public boolean hasGuidePosts() { + return ((bitField0_ & 0x00000002) == 0x00000002); } /** - * repeated bytes values = 2; + * optional bytes guidePosts = 2; */ - public com.google.protobuf.ByteString getValues(int index) { - return values_.get(index); + public com.google.protobuf.ByteString getGuidePosts() { + return guidePosts_; } // optional int64 guidePostsByteCount = 3; @@ -2082,7 +2080,7 @@ public com.google.protobuf.ByteString getValues(int index) { * optional int64 guidePostsByteCount = 3; */ public boolean hasGuidePostsByteCount() { - return ((bitField0_ & 0x00000002) == 0x00000002); + return ((bitField0_ & 0x00000004) == 0x00000004); } /** * optional int64 guidePostsByteCount = 3; @@ -2098,7 +2096,7 @@ public long getGuidePostsByteCount() { * optional int64 keyBytesCount = 4; */ public boolean hasKeyBytesCount() { - return ((bitField0_ & 0x00000004) == 0x00000004); + return ((bitField0_ & 0x00000008) == 0x00000008); } /** * optional int64 keyBytesCount = 4; @@ -2114,7 +2112,7 @@ public long getKeyBytesCount() { * optional int32 guidePostsCount = 5; */ public boolean hasGuidePostsCount() { - return ((bitField0_ & 0x00000008) == 0x00000008); + return ((bitField0_ & 0x00000010) == 0x00000010); } /** * optional int32 guidePostsCount = 5; @@ -2130,7 +2128,7 @@ public int getGuidePostsCount() { * optional .PGuidePosts pGuidePosts = 6; */ public boolean hasPGuidePosts() { - return ((bitField0_ & 0x00000010) == 0x00000010); + return ((bitField0_ & 0x00000020) == 0x00000020); } /** * optional .PGuidePosts pGuidePosts = 6; @@ -2145,13 +2143,30 @@ public org.apache.phoenix.coprocessor.generated.PGuidePostsProtos.PGuidePostsOrB return pGuidePosts_; } + // optional int32 maxLength = 7; + public static final int MAXLENGTH_FIELD_NUMBER = 7; + private int maxLength_; + /** + * optional int32 maxLength = 7; + */ + public boolean hasMaxLength() { + return ((bitField0_ & 0x00000040) == 0x00000040); + } + /** + * optional int32 maxLength = 7; + */ + public int getMaxLength() { + return maxLength_; + } + private void initFields() { key_ = com.google.protobuf.ByteString.EMPTY; - values_ = java.util.Collections.emptyList(); + guidePosts_ = com.google.protobuf.ByteString.EMPTY; guidePostsByteCount_ = 0L; keyBytesCount_ = 0L; guidePostsCount_ = 0; pGuidePosts_ = org.apache.phoenix.coprocessor.generated.PGuidePostsProtos.PGuidePosts.getDefaultInstance(); + maxLength_ = 0; } private byte memoizedIsInitialized = -1; public final boolean isInitialized() { @@ -2172,21 +2187,24 @@ public void writeTo(com.google.protobuf.CodedOutputStream output) if (((bitField0_ & 0x00000001) == 0x00000001)) { output.writeBytes(1, key_); } - for (int i = 0; i < values_.size(); i++) { - output.writeBytes(2, values_.get(i)); - } if (((bitField0_ & 0x00000002) == 0x00000002)) { - output.writeInt64(3, guidePostsByteCount_); + output.writeBytes(2, guidePosts_); } if (((bitField0_ & 0x00000004) == 0x00000004)) { - output.writeInt64(4, keyBytesCount_); + output.writeInt64(3, guidePostsByteCount_); } if (((bitField0_ & 0x00000008) == 0x00000008)) { - output.writeInt32(5, guidePostsCount_); + output.writeInt64(4, keyBytesCount_); } if (((bitField0_ & 0x00000010) == 0x00000010)) { + output.writeInt32(5, guidePostsCount_); + } + if (((bitField0_ & 0x00000020) == 0x00000020)) { output.writeMessage(6, pGuidePosts_); } + if (((bitField0_ & 0x00000040) == 0x00000040)) { + output.writeInt32(7, maxLength_); + } getUnknownFields().writeTo(output); } @@ -2200,31 +2218,30 @@ public int getSerializedSize() { size += com.google.protobuf.CodedOutputStream .computeBytesSize(1, key_); } - { - int dataSize = 0; - for (int i = 0; i < values_.size(); i++) { - dataSize += com.google.protobuf.CodedOutputStream - .computeBytesSizeNoTag(values_.get(i)); - } - size += dataSize; - size += 1 * getValuesList().size(); - } if (((bitField0_ & 0x00000002) == 0x00000002)) { size += com.google.protobuf.CodedOutputStream - .computeInt64Size(3, guidePostsByteCount_); + .computeBytesSize(2, guidePosts_); } if (((bitField0_ & 0x00000004) == 0x00000004)) { size += com.google.protobuf.CodedOutputStream - .computeInt64Size(4, keyBytesCount_); + .computeInt64Size(3, guidePostsByteCount_); } if (((bitField0_ & 0x00000008) == 0x00000008)) { size += com.google.protobuf.CodedOutputStream - .computeInt32Size(5, guidePostsCount_); + .computeInt64Size(4, keyBytesCount_); } if (((bitField0_ & 0x00000010) == 0x00000010)) { + size += com.google.protobuf.CodedOutputStream + .computeInt32Size(5, guidePostsCount_); + } + if (((bitField0_ & 0x00000020) == 0x00000020)) { size += com.google.protobuf.CodedOutputStream .computeMessageSize(6, pGuidePosts_); } + if (((bitField0_ & 0x00000040) == 0x00000040)) { + size += com.google.protobuf.CodedOutputStream + .computeInt32Size(7, maxLength_); + } size += getUnknownFields().getSerializedSize(); memoizedSerializedSize = size; return size; @@ -2253,8 +2270,11 @@ public boolean equals(final java.lang.Object obj) { result = result && getKey() .equals(other.getKey()); } - result = result && getValuesList() - .equals(other.getValuesList()); + result = result && (hasGuidePosts() == other.hasGuidePosts()); + if (hasGuidePosts()) { + result = result && getGuidePosts() + .equals(other.getGuidePosts()); + } result = result && (hasGuidePostsByteCount() == other.hasGuidePostsByteCount()); if (hasGuidePostsByteCount()) { result = result && (getGuidePostsByteCount() @@ -2275,6 +2295,11 @@ public boolean equals(final java.lang.Object obj) { result = result && getPGuidePosts() .equals(other.getPGuidePosts()); } + result = result && (hasMaxLength() == other.hasMaxLength()); + if (hasMaxLength()) { + result = result && (getMaxLength() + == other.getMaxLength()); + } result = result && getUnknownFields().equals(other.getUnknownFields()); return result; @@ -2292,9 +2317,9 @@ public int hashCode() { hash = (37 * hash) + KEY_FIELD_NUMBER; hash = (53 * hash) + getKey().hashCode(); } - if (getValuesCount() > 0) { - hash = (37 * hash) + VALUES_FIELD_NUMBER; - hash = (53 * hash) + getValuesList().hashCode(); + if (hasGuidePosts()) { + hash = (37 * hash) + GUIDEPOSTS_FIELD_NUMBER; + hash = (53 * hash) + getGuidePosts().hashCode(); } if (hasGuidePostsByteCount()) { hash = (37 * hash) + GUIDEPOSTSBYTECOUNT_FIELD_NUMBER; @@ -2312,6 +2337,10 @@ public int hashCode() { hash = (37 * hash) + PGUIDEPOSTS_FIELD_NUMBER; hash = (53 * hash) + getPGuidePosts().hashCode(); } + if (hasMaxLength()) { + hash = (37 * hash) + MAXLENGTH_FIELD_NUMBER; + hash = (53 * hash) + getMaxLength(); + } hash = (29 * hash) + getUnknownFields().hashCode(); memoizedHashCode = hash; return hash; @@ -2424,7 +2453,7 @@ public Builder clear() { super.clear(); key_ = com.google.protobuf.ByteString.EMPTY; bitField0_ = (bitField0_ & ~0x00000001); - values_ = java.util.Collections.emptyList(); + guidePosts_ = com.google.protobuf.ByteString.EMPTY; bitField0_ = (bitField0_ & ~0x00000002); guidePostsByteCount_ = 0L; bitField0_ = (bitField0_ & ~0x00000004); @@ -2438,6 +2467,8 @@ public Builder clear() { pGuidePostsBuilder_.clear(); } bitField0_ = (bitField0_ & ~0x00000020); + maxLength_ = 0; + bitField0_ = (bitField0_ & ~0x00000040); return this; } @@ -2470,31 +2501,34 @@ public org.apache.phoenix.coprocessor.generated.PTableProtos.PTableStats buildPa to_bitField0_ |= 0x00000001; } result.key_ = key_; - if (((bitField0_ & 0x00000002) == 0x00000002)) { - values_ = java.util.Collections.unmodifiableList(values_); - bitField0_ = (bitField0_ & ~0x00000002); + if (((from_bitField0_ & 0x00000002) == 0x00000002)) { + to_bitField0_ |= 0x00000002; } - result.values_ = values_; + result.guidePosts_ = guidePosts_; if (((from_bitField0_ & 0x00000004) == 0x00000004)) { - to_bitField0_ |= 0x00000002; + to_bitField0_ |= 0x00000004; } result.guidePostsByteCount_ = guidePostsByteCount_; if (((from_bitField0_ & 0x00000008) == 0x00000008)) { - to_bitField0_ |= 0x00000004; + to_bitField0_ |= 0x00000008; } result.keyBytesCount_ = keyBytesCount_; if (((from_bitField0_ & 0x00000010) == 0x00000010)) { - to_bitField0_ |= 0x00000008; + to_bitField0_ |= 0x00000010; } result.guidePostsCount_ = guidePostsCount_; if (((from_bitField0_ & 0x00000020) == 0x00000020)) { - to_bitField0_ |= 0x00000010; + to_bitField0_ |= 0x00000020; } if (pGuidePostsBuilder_ == null) { result.pGuidePosts_ = pGuidePosts_; } else { result.pGuidePosts_ = pGuidePostsBuilder_.build(); } + if (((from_bitField0_ & 0x00000040) == 0x00000040)) { + to_bitField0_ |= 0x00000040; + } + result.maxLength_ = maxLength_; result.bitField0_ = to_bitField0_; onBuilt(); return result; @@ -2514,15 +2548,8 @@ public Builder mergeFrom(org.apache.phoenix.coprocessor.generated.PTableProtos.P if (other.hasKey()) { setKey(other.getKey()); } - if (!other.values_.isEmpty()) { - if (values_.isEmpty()) { - values_ = other.values_; - bitField0_ = (bitField0_ & ~0x00000002); - } else { - ensureValuesIsMutable(); - values_.addAll(other.values_); - } - onChanged(); + if (other.hasGuidePosts()) { + setGuidePosts(other.getGuidePosts()); } if (other.hasGuidePostsByteCount()) { setGuidePostsByteCount(other.getGuidePostsByteCount()); @@ -2536,6 +2563,9 @@ public Builder mergeFrom(org.apache.phoenix.coprocessor.generated.PTableProtos.P if (other.hasPGuidePosts()) { mergePGuidePosts(other.getPGuidePosts()); } + if (other.hasMaxLength()) { + setMaxLength(other.getMaxLength()); + } this.mergeUnknownFields(other.getUnknownFields()); return this; } @@ -2603,74 +2633,38 @@ public Builder clearKey() { return this; } - // repeated bytes values = 2; - private java.util.List values_ = java.util.Collections.emptyList(); - private void ensureValuesIsMutable() { - if (!((bitField0_ & 0x00000002) == 0x00000002)) { - values_ = new java.util.ArrayList(values_); - bitField0_ |= 0x00000002; - } - } - /** - * repeated bytes values = 2; - */ - public java.util.List - getValuesList() { - return java.util.Collections.unmodifiableList(values_); - } + // optional bytes guidePosts = 2; + private com.google.protobuf.ByteString guidePosts_ = com.google.protobuf.ByteString.EMPTY; /** - * repeated bytes values = 2; + * optional bytes guidePosts = 2; */ - public int getValuesCount() { - return values_.size(); + public boolean hasGuidePosts() { + return ((bitField0_ & 0x00000002) == 0x00000002); } /** - * repeated bytes values = 2; + * optional bytes guidePosts = 2; */ - public com.google.protobuf.ByteString getValues(int index) { - return values_.get(index); + public com.google.protobuf.ByteString getGuidePosts() { + return guidePosts_; } /** - * repeated bytes values = 2; + * optional bytes guidePosts = 2; */ - public Builder setValues( - int index, com.google.protobuf.ByteString value) { + public Builder setGuidePosts(com.google.protobuf.ByteString value) { if (value == null) { throw new NullPointerException(); } - ensureValuesIsMutable(); - values_.set(index, value); - onChanged(); - return this; - } - /** - * repeated bytes values = 2; - */ - public Builder addValues(com.google.protobuf.ByteString value) { - if (value == null) { - throw new NullPointerException(); - } - ensureValuesIsMutable(); - values_.add(value); - onChanged(); - return this; - } - /** - * repeated bytes values = 2; - */ - public Builder addAllValues( - java.lang.Iterable values) { - ensureValuesIsMutable(); - super.addAll(values, values_); + bitField0_ |= 0x00000002; + guidePosts_ = value; onChanged(); return this; } /** - * repeated bytes values = 2; + * optional bytes guidePosts = 2; */ - public Builder clearValues() { - values_ = java.util.Collections.emptyList(); + public Builder clearGuidePosts() { bitField0_ = (bitField0_ & ~0x00000002); + guidePosts_ = getDefaultInstance().getGuidePosts(); onChanged(); return this; } @@ -2891,6 +2885,39 @@ public org.apache.phoenix.coprocessor.generated.PGuidePostsProtos.PGuidePostsOrB return pGuidePostsBuilder_; } + // optional int32 maxLength = 7; + private int maxLength_ ; + /** + * optional int32 maxLength = 7; + */ + public boolean hasMaxLength() { + return ((bitField0_ & 0x00000040) == 0x00000040); + } + /** + * optional int32 maxLength = 7; + */ + public int getMaxLength() { + return maxLength_; + } + /** + * optional int32 maxLength = 7; + */ + public Builder setMaxLength(int value) { + bitField0_ |= 0x00000040; + maxLength_ = value; + onChanged(); + return this; + } + /** + * optional int32 maxLength = 7; + */ + public Builder clearMaxLength() { + bitField0_ = (bitField0_ & ~0x00000040); + maxLength_ = 0; + onChanged(); + return this; + } + // @@protoc_insertion_point(builder_scope:PTableStats) } @@ -6826,31 +6853,31 @@ public Builder clearTransactional() { "\006 \002(\010\022\020\n\010position\030\007 \002(\005\022\021\n\tsortOrder\030\010 \002" + "(\005\022\021\n\tarraySize\030\t \001(\005\022\024\n\014viewConstant\030\n " + "\001(\014\022\026\n\016viewReferenced\030\013 \001(\010\022\022\n\nexpressio" + - "n\030\014 \001(\t\022\026\n\016isRowTimestamp\030\r \001(\010\"\232\001\n\013PTab" + - "leStats\022\013\n\003key\030\001 \002(\014\022\016\n\006values\030\002 \003(\014\022\033\n\023" + - "guidePostsByteCount\030\003 \001(\003\022\025\n\rkeyBytesCou", - "nt\030\004 \001(\003\022\027\n\017guidePostsCount\030\005 \001(\005\022!\n\013pGu" + - "idePosts\030\006 \001(\0132\014.PGuidePosts\"\206\005\n\006PTable\022" + - "\027\n\017schemaNameBytes\030\001 \002(\014\022\026\n\016tableNameByt" + - "es\030\002 \002(\014\022\036\n\ttableType\030\003 \002(\0162\013.PTableType" + - "\022\022\n\nindexState\030\004 \001(\t\022\026\n\016sequenceNumber\030\005" + - " \002(\003\022\021\n\ttimeStamp\030\006 \002(\003\022\023\n\013pkNameBytes\030\007" + - " \001(\014\022\021\n\tbucketNum\030\010 \002(\005\022\031\n\007columns\030\t \003(\013" + - "2\010.PColumn\022\030\n\007indexes\030\n \003(\0132\007.PTable\022\027\n\017" + - "isImmutableRows\030\013 \002(\010\022 \n\nguidePosts\030\014 \003(" + - "\0132\014.PTableStats\022\032\n\022dataTableNameBytes\030\r ", - "\001(\014\022\031\n\021defaultFamilyName\030\016 \001(\014\022\022\n\ndisabl" + - "eWAL\030\017 \002(\010\022\023\n\013multiTenant\030\020 \002(\010\022\020\n\010viewT" + - "ype\030\021 \001(\014\022\025\n\rviewStatement\030\022 \001(\014\022\025\n\rphys" + - "icalNames\030\023 \003(\014\022\020\n\010tenantId\030\024 \001(\014\022\023\n\013vie" + - "wIndexId\030\025 \001(\005\022\021\n\tindexType\030\026 \001(\014\022\026\n\016sta" + - "tsTimeStamp\030\027 \001(\003\022\022\n\nstoreNulls\030\030 \001(\010\022\027\n" + - "\017baseColumnCount\030\031 \001(\005\022\036\n\026rowKeyOrderOpt" + - "imizable\030\032 \001(\010\022\025\n\rtransactional\030\033 \001(\010*A\n" + - "\nPTableType\022\n\n\006SYSTEM\020\000\022\010\n\004USER\020\001\022\010\n\004VIE" + - "W\020\002\022\t\n\005INDEX\020\003\022\010\n\004JOIN\020\004B@\n(org.apache.p", - "hoenix.coprocessor.generatedB\014PTableProt" + - "osH\001\210\001\001\240\001\001" + "n\030\014 \001(\t\022\026\n\016isRowTimestamp\030\r \001(\010\"\261\001\n\013PTab" + + "leStats\022\013\n\003key\030\001 \002(\014\022\022\n\nguidePosts\030\002 \001(\014" + + "\022\033\n\023guidePostsByteCount\030\003 \001(\003\022\025\n\rkeyByte", + "sCount\030\004 \001(\003\022\027\n\017guidePostsCount\030\005 \001(\005\022!\n" + + "\013pGuidePosts\030\006 \001(\0132\014.PGuidePosts\022\021\n\tmaxL" + + "ength\030\007 \001(\005\"\206\005\n\006PTable\022\027\n\017schemaNameByte" + + "s\030\001 \002(\014\022\026\n\016tableNameBytes\030\002 \002(\014\022\036\n\ttable" + + "Type\030\003 \002(\0162\013.PTableType\022\022\n\nindexState\030\004 " + + "\001(\t\022\026\n\016sequenceNumber\030\005 \002(\003\022\021\n\ttimeStamp" + + "\030\006 \002(\003\022\023\n\013pkNameBytes\030\007 \001(\014\022\021\n\tbucketNum" + + "\030\010 \002(\005\022\031\n\007columns\030\t \003(\0132\010.PColumn\022\030\n\007ind" + + "exes\030\n \003(\0132\007.PTable\022\027\n\017isImmutableRows\030\013" + + " \002(\010\022 \n\nguidePosts\030\014 \003(\0132\014.PTableStats\022\032", + "\n\022dataTableNameBytes\030\r \001(\014\022\031\n\021defaultFam" + + "ilyName\030\016 \001(\014\022\022\n\ndisableWAL\030\017 \002(\010\022\023\n\013mul" + + "tiTenant\030\020 \002(\010\022\020\n\010viewType\030\021 \001(\014\022\025\n\rview" + + "Statement\030\022 \001(\014\022\025\n\rphysicalNames\030\023 \003(\014\022\020" + + "\n\010tenantId\030\024 \001(\014\022\023\n\013viewIndexId\030\025 \001(\005\022\021\n" + + "\tindexType\030\026 \001(\014\022\026\n\016statsTimeStamp\030\027 \001(\003" + + "\022\022\n\nstoreNulls\030\030 \001(\010\022\027\n\017baseColumnCount\030" + + "\031 \001(\005\022\036\n\026rowKeyOrderOptimizable\030\032 \001(\010\022\025\n" + + "\rtransactional\030\033 \001(\010*A\n\nPTableType\022\n\n\006SY" + + "STEM\020\000\022\010\n\004USER\020\001\022\010\n\004VIEW\020\002\022\t\n\005INDEX\020\003\022\010\n", + "\004JOIN\020\004B@\n(org.apache.phoenix.coprocesso" + + "r.generatedB\014PTableProtosH\001\210\001\001\240\001\001" }; com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner = new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() { @@ -6868,7 +6895,7 @@ public com.google.protobuf.ExtensionRegistry assignDescriptors( internal_static_PTableStats_fieldAccessorTable = new com.google.protobuf.GeneratedMessage.FieldAccessorTable( internal_static_PTableStats_descriptor, - new java.lang.String[] { "Key", "Values", "GuidePostsByteCount", "KeyBytesCount", "GuidePostsCount", "PGuidePosts", }); + new java.lang.String[] { "Key", "GuidePosts", "GuidePostsByteCount", "KeyBytesCount", "GuidePostsCount", "PGuidePosts", "MaxLength", }); internal_static_PTable_descriptor = getDescriptor().getMessageTypes().get(2); internal_static_PTable_fieldAccessorTable = new diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/StatCollectorProtos.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/StatCollectorProtos.java deleted file mode 100644 index 5008f22a303..00000000000 --- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/StatCollectorProtos.java +++ /dev/null @@ -1,1269 +0,0 @@ -// Generated by the protocol buffer compiler. DO NOT EDIT! -// source: StatisticsCollect.proto - -package org.apache.phoenix.coprocessor.generated; - -public final class StatCollectorProtos { - private StatCollectorProtos() {} - public static void registerAllExtensions( - com.google.protobuf.ExtensionRegistry registry) { - } - public interface StatCollectRequestOrBuilder - extends com.google.protobuf.MessageOrBuilder { - - // optional bytes startRow = 1; - /** - * optional bytes startRow = 1; - */ - boolean hasStartRow(); - /** - * optional bytes startRow = 1; - */ - com.google.protobuf.ByteString getStartRow(); - - // optional bytes stopRow = 2; - /** - * optional bytes stopRow = 2; - */ - boolean hasStopRow(); - /** - * optional bytes stopRow = 2; - */ - com.google.protobuf.ByteString getStopRow(); - } - /** - * Protobuf type {@code StatCollectRequest} - */ - public static final class StatCollectRequest extends - com.google.protobuf.GeneratedMessage - implements StatCollectRequestOrBuilder { - // Use StatCollectRequest.newBuilder() to construct. - private StatCollectRequest(com.google.protobuf.GeneratedMessage.Builder builder) { - super(builder); - this.unknownFields = builder.getUnknownFields(); - } - private StatCollectRequest(boolean noInit) { this.unknownFields = com.google.protobuf.UnknownFieldSet.getDefaultInstance(); } - - private static final StatCollectRequest defaultInstance; - public static StatCollectRequest getDefaultInstance() { - return defaultInstance; - } - - public StatCollectRequest getDefaultInstanceForType() { - return defaultInstance; - } - - private final com.google.protobuf.UnknownFieldSet unknownFields; - @java.lang.Override - public final com.google.protobuf.UnknownFieldSet - getUnknownFields() { - return this.unknownFields; - } - private StatCollectRequest( - com.google.protobuf.CodedInputStream input, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws com.google.protobuf.InvalidProtocolBufferException { - initFields(); - int mutable_bitField0_ = 0; - com.google.protobuf.UnknownFieldSet.Builder unknownFields = - com.google.protobuf.UnknownFieldSet.newBuilder(); - try { - boolean done = false; - while (!done) { - int tag = input.readTag(); - switch (tag) { - case 0: - done = true; - break; - default: { - if (!parseUnknownField(input, unknownFields, - extensionRegistry, tag)) { - done = true; - } - break; - } - case 10: { - bitField0_ |= 0x00000001; - startRow_ = input.readBytes(); - break; - } - case 18: { - bitField0_ |= 0x00000002; - stopRow_ = input.readBytes(); - break; - } - } - } - } catch (com.google.protobuf.InvalidProtocolBufferException e) { - throw e.setUnfinishedMessage(this); - } catch (java.io.IOException e) { - throw new com.google.protobuf.InvalidProtocolBufferException( - e.getMessage()).setUnfinishedMessage(this); - } finally { - this.unknownFields = unknownFields.build(); - makeExtensionsImmutable(); - } - } - public static final com.google.protobuf.Descriptors.Descriptor - getDescriptor() { - return org.apache.phoenix.coprocessor.generated.StatCollectorProtos.internal_static_StatCollectRequest_descriptor; - } - - protected com.google.protobuf.GeneratedMessage.FieldAccessorTable - internalGetFieldAccessorTable() { - return org.apache.phoenix.coprocessor.generated.StatCollectorProtos.internal_static_StatCollectRequest_fieldAccessorTable - .ensureFieldAccessorsInitialized( - org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectRequest.class, org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectRequest.Builder.class); - } - - public static com.google.protobuf.Parser PARSER = - new com.google.protobuf.AbstractParser() { - public StatCollectRequest parsePartialFrom( - com.google.protobuf.CodedInputStream input, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws com.google.protobuf.InvalidProtocolBufferException { - return new StatCollectRequest(input, extensionRegistry); - } - }; - - @java.lang.Override - public com.google.protobuf.Parser getParserForType() { - return PARSER; - } - - private int bitField0_; - // optional bytes startRow = 1; - public static final int STARTROW_FIELD_NUMBER = 1; - private com.google.protobuf.ByteString startRow_; - /** - * optional bytes startRow = 1; - */ - public boolean hasStartRow() { - return ((bitField0_ & 0x00000001) == 0x00000001); - } - /** - * optional bytes startRow = 1; - */ - public com.google.protobuf.ByteString getStartRow() { - return startRow_; - } - - // optional bytes stopRow = 2; - public static final int STOPROW_FIELD_NUMBER = 2; - private com.google.protobuf.ByteString stopRow_; - /** - * optional bytes stopRow = 2; - */ - public boolean hasStopRow() { - return ((bitField0_ & 0x00000002) == 0x00000002); - } - /** - * optional bytes stopRow = 2; - */ - public com.google.protobuf.ByteString getStopRow() { - return stopRow_; - } - - private void initFields() { - startRow_ = com.google.protobuf.ByteString.EMPTY; - stopRow_ = com.google.protobuf.ByteString.EMPTY; - } - private byte memoizedIsInitialized = -1; - public final boolean isInitialized() { - byte isInitialized = memoizedIsInitialized; - if (isInitialized != -1) return isInitialized == 1; - - memoizedIsInitialized = 1; - return true; - } - - public void writeTo(com.google.protobuf.CodedOutputStream output) - throws java.io.IOException { - getSerializedSize(); - if (((bitField0_ & 0x00000001) == 0x00000001)) { - output.writeBytes(1, startRow_); - } - if (((bitField0_ & 0x00000002) == 0x00000002)) { - output.writeBytes(2, stopRow_); - } - getUnknownFields().writeTo(output); - } - - private int memoizedSerializedSize = -1; - public int getSerializedSize() { - int size = memoizedSerializedSize; - if (size != -1) return size; - - size = 0; - if (((bitField0_ & 0x00000001) == 0x00000001)) { - size += com.google.protobuf.CodedOutputStream - .computeBytesSize(1, startRow_); - } - if (((bitField0_ & 0x00000002) == 0x00000002)) { - size += com.google.protobuf.CodedOutputStream - .computeBytesSize(2, stopRow_); - } - size += getUnknownFields().getSerializedSize(); - memoizedSerializedSize = size; - return size; - } - - private static final long serialVersionUID = 0L; - @java.lang.Override - protected java.lang.Object writeReplace() - throws java.io.ObjectStreamException { - return super.writeReplace(); - } - - @java.lang.Override - public boolean equals(final java.lang.Object obj) { - if (obj == this) { - return true; - } - if (!(obj instanceof org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectRequest)) { - return super.equals(obj); - } - org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectRequest other = (org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectRequest) obj; - - boolean result = true; - result = result && (hasStartRow() == other.hasStartRow()); - if (hasStartRow()) { - result = result && getStartRow() - .equals(other.getStartRow()); - } - result = result && (hasStopRow() == other.hasStopRow()); - if (hasStopRow()) { - result = result && getStopRow() - .equals(other.getStopRow()); - } - result = result && - getUnknownFields().equals(other.getUnknownFields()); - return result; - } - - private int memoizedHashCode = 0; - @java.lang.Override - public int hashCode() { - if (memoizedHashCode != 0) { - return memoizedHashCode; - } - int hash = 41; - hash = (19 * hash) + getDescriptorForType().hashCode(); - if (hasStartRow()) { - hash = (37 * hash) + STARTROW_FIELD_NUMBER; - hash = (53 * hash) + getStartRow().hashCode(); - } - if (hasStopRow()) { - hash = (37 * hash) + STOPROW_FIELD_NUMBER; - hash = (53 * hash) + getStopRow().hashCode(); - } - hash = (29 * hash) + getUnknownFields().hashCode(); - memoizedHashCode = hash; - return hash; - } - - public static org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectRequest parseFrom( - com.google.protobuf.ByteString data) - throws com.google.protobuf.InvalidProtocolBufferException { - return PARSER.parseFrom(data); - } - public static org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectRequest parseFrom( - com.google.protobuf.ByteString data, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws com.google.protobuf.InvalidProtocolBufferException { - return PARSER.parseFrom(data, extensionRegistry); - } - public static org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectRequest parseFrom(byte[] data) - throws com.google.protobuf.InvalidProtocolBufferException { - return PARSER.parseFrom(data); - } - public static org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectRequest parseFrom( - byte[] data, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws com.google.protobuf.InvalidProtocolBufferException { - return PARSER.parseFrom(data, extensionRegistry); - } - public static org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectRequest parseFrom(java.io.InputStream input) - throws java.io.IOException { - return PARSER.parseFrom(input); - } - public static org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectRequest parseFrom( - java.io.InputStream input, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws java.io.IOException { - return PARSER.parseFrom(input, extensionRegistry); - } - public static org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectRequest parseDelimitedFrom(java.io.InputStream input) - throws java.io.IOException { - return PARSER.parseDelimitedFrom(input); - } - public static org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectRequest parseDelimitedFrom( - java.io.InputStream input, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws java.io.IOException { - return PARSER.parseDelimitedFrom(input, extensionRegistry); - } - public static org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectRequest parseFrom( - com.google.protobuf.CodedInputStream input) - throws java.io.IOException { - return PARSER.parseFrom(input); - } - public static org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectRequest parseFrom( - com.google.protobuf.CodedInputStream input, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws java.io.IOException { - return PARSER.parseFrom(input, extensionRegistry); - } - - public static Builder newBuilder() { return Builder.create(); } - public Builder newBuilderForType() { return newBuilder(); } - public static Builder newBuilder(org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectRequest prototype) { - return newBuilder().mergeFrom(prototype); - } - public Builder toBuilder() { return newBuilder(this); } - - @java.lang.Override - protected Builder newBuilderForType( - com.google.protobuf.GeneratedMessage.BuilderParent parent) { - Builder builder = new Builder(parent); - return builder; - } - /** - * Protobuf type {@code StatCollectRequest} - */ - public static final class Builder extends - com.google.protobuf.GeneratedMessage.Builder - implements org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectRequestOrBuilder { - public static final com.google.protobuf.Descriptors.Descriptor - getDescriptor() { - return org.apache.phoenix.coprocessor.generated.StatCollectorProtos.internal_static_StatCollectRequest_descriptor; - } - - protected com.google.protobuf.GeneratedMessage.FieldAccessorTable - internalGetFieldAccessorTable() { - return org.apache.phoenix.coprocessor.generated.StatCollectorProtos.internal_static_StatCollectRequest_fieldAccessorTable - .ensureFieldAccessorsInitialized( - org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectRequest.class, org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectRequest.Builder.class); - } - - // Construct using org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectRequest.newBuilder() - private Builder() { - maybeForceBuilderInitialization(); - } - - private Builder( - com.google.protobuf.GeneratedMessage.BuilderParent parent) { - super(parent); - maybeForceBuilderInitialization(); - } - private void maybeForceBuilderInitialization() { - if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) { - } - } - private static Builder create() { - return new Builder(); - } - - public Builder clear() { - super.clear(); - startRow_ = com.google.protobuf.ByteString.EMPTY; - bitField0_ = (bitField0_ & ~0x00000001); - stopRow_ = com.google.protobuf.ByteString.EMPTY; - bitField0_ = (bitField0_ & ~0x00000002); - return this; - } - - public Builder clone() { - return create().mergeFrom(buildPartial()); - } - - public com.google.protobuf.Descriptors.Descriptor - getDescriptorForType() { - return org.apache.phoenix.coprocessor.generated.StatCollectorProtos.internal_static_StatCollectRequest_descriptor; - } - - public org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectRequest getDefaultInstanceForType() { - return org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectRequest.getDefaultInstance(); - } - - public org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectRequest build() { - org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectRequest result = buildPartial(); - if (!result.isInitialized()) { - throw newUninitializedMessageException(result); - } - return result; - } - - public org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectRequest buildPartial() { - org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectRequest result = new org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectRequest(this); - int from_bitField0_ = bitField0_; - int to_bitField0_ = 0; - if (((from_bitField0_ & 0x00000001) == 0x00000001)) { - to_bitField0_ |= 0x00000001; - } - result.startRow_ = startRow_; - if (((from_bitField0_ & 0x00000002) == 0x00000002)) { - to_bitField0_ |= 0x00000002; - } - result.stopRow_ = stopRow_; - result.bitField0_ = to_bitField0_; - onBuilt(); - return result; - } - - public Builder mergeFrom(com.google.protobuf.Message other) { - if (other instanceof org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectRequest) { - return mergeFrom((org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectRequest)other); - } else { - super.mergeFrom(other); - return this; - } - } - - public Builder mergeFrom(org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectRequest other) { - if (other == org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectRequest.getDefaultInstance()) return this; - if (other.hasStartRow()) { - setStartRow(other.getStartRow()); - } - if (other.hasStopRow()) { - setStopRow(other.getStopRow()); - } - this.mergeUnknownFields(other.getUnknownFields()); - return this; - } - - public final boolean isInitialized() { - return true; - } - - public Builder mergeFrom( - com.google.protobuf.CodedInputStream input, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws java.io.IOException { - org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectRequest parsedMessage = null; - try { - parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry); - } catch (com.google.protobuf.InvalidProtocolBufferException e) { - parsedMessage = (org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectRequest) e.getUnfinishedMessage(); - throw e; - } finally { - if (parsedMessage != null) { - mergeFrom(parsedMessage); - } - } - return this; - } - private int bitField0_; - - // optional bytes startRow = 1; - private com.google.protobuf.ByteString startRow_ = com.google.protobuf.ByteString.EMPTY; - /** - * optional bytes startRow = 1; - */ - public boolean hasStartRow() { - return ((bitField0_ & 0x00000001) == 0x00000001); - } - /** - * optional bytes startRow = 1; - */ - public com.google.protobuf.ByteString getStartRow() { - return startRow_; - } - /** - * optional bytes startRow = 1; - */ - public Builder setStartRow(com.google.protobuf.ByteString value) { - if (value == null) { - throw new NullPointerException(); - } - bitField0_ |= 0x00000001; - startRow_ = value; - onChanged(); - return this; - } - /** - * optional bytes startRow = 1; - */ - public Builder clearStartRow() { - bitField0_ = (bitField0_ & ~0x00000001); - startRow_ = getDefaultInstance().getStartRow(); - onChanged(); - return this; - } - - // optional bytes stopRow = 2; - private com.google.protobuf.ByteString stopRow_ = com.google.protobuf.ByteString.EMPTY; - /** - * optional bytes stopRow = 2; - */ - public boolean hasStopRow() { - return ((bitField0_ & 0x00000002) == 0x00000002); - } - /** - * optional bytes stopRow = 2; - */ - public com.google.protobuf.ByteString getStopRow() { - return stopRow_; - } - /** - * optional bytes stopRow = 2; - */ - public Builder setStopRow(com.google.protobuf.ByteString value) { - if (value == null) { - throw new NullPointerException(); - } - bitField0_ |= 0x00000002; - stopRow_ = value; - onChanged(); - return this; - } - /** - * optional bytes stopRow = 2; - */ - public Builder clearStopRow() { - bitField0_ = (bitField0_ & ~0x00000002); - stopRow_ = getDefaultInstance().getStopRow(); - onChanged(); - return this; - } - - // @@protoc_insertion_point(builder_scope:StatCollectRequest) - } - - static { - defaultInstance = new StatCollectRequest(true); - defaultInstance.initFields(); - } - - // @@protoc_insertion_point(class_scope:StatCollectRequest) - } - - public interface StatCollectResponseOrBuilder - extends com.google.protobuf.MessageOrBuilder { - - // required uint64 rowsScanned = 1; - /** - * required uint64 rowsScanned = 1; - */ - boolean hasRowsScanned(); - /** - * required uint64 rowsScanned = 1; - */ - long getRowsScanned(); - } - /** - * Protobuf type {@code StatCollectResponse} - */ - public static final class StatCollectResponse extends - com.google.protobuf.GeneratedMessage - implements StatCollectResponseOrBuilder { - // Use StatCollectResponse.newBuilder() to construct. - private StatCollectResponse(com.google.protobuf.GeneratedMessage.Builder builder) { - super(builder); - this.unknownFields = builder.getUnknownFields(); - } - private StatCollectResponse(boolean noInit) { this.unknownFields = com.google.protobuf.UnknownFieldSet.getDefaultInstance(); } - - private static final StatCollectResponse defaultInstance; - public static StatCollectResponse getDefaultInstance() { - return defaultInstance; - } - - public StatCollectResponse getDefaultInstanceForType() { - return defaultInstance; - } - - private final com.google.protobuf.UnknownFieldSet unknownFields; - @java.lang.Override - public final com.google.protobuf.UnknownFieldSet - getUnknownFields() { - return this.unknownFields; - } - private StatCollectResponse( - com.google.protobuf.CodedInputStream input, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws com.google.protobuf.InvalidProtocolBufferException { - initFields(); - int mutable_bitField0_ = 0; - com.google.protobuf.UnknownFieldSet.Builder unknownFields = - com.google.protobuf.UnknownFieldSet.newBuilder(); - try { - boolean done = false; - while (!done) { - int tag = input.readTag(); - switch (tag) { - case 0: - done = true; - break; - default: { - if (!parseUnknownField(input, unknownFields, - extensionRegistry, tag)) { - done = true; - } - break; - } - case 8: { - bitField0_ |= 0x00000001; - rowsScanned_ = input.readUInt64(); - break; - } - } - } - } catch (com.google.protobuf.InvalidProtocolBufferException e) { - throw e.setUnfinishedMessage(this); - } catch (java.io.IOException e) { - throw new com.google.protobuf.InvalidProtocolBufferException( - e.getMessage()).setUnfinishedMessage(this); - } finally { - this.unknownFields = unknownFields.build(); - makeExtensionsImmutable(); - } - } - public static final com.google.protobuf.Descriptors.Descriptor - getDescriptor() { - return org.apache.phoenix.coprocessor.generated.StatCollectorProtos.internal_static_StatCollectResponse_descriptor; - } - - protected com.google.protobuf.GeneratedMessage.FieldAccessorTable - internalGetFieldAccessorTable() { - return org.apache.phoenix.coprocessor.generated.StatCollectorProtos.internal_static_StatCollectResponse_fieldAccessorTable - .ensureFieldAccessorsInitialized( - org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectResponse.class, org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectResponse.Builder.class); - } - - public static com.google.protobuf.Parser PARSER = - new com.google.protobuf.AbstractParser() { - public StatCollectResponse parsePartialFrom( - com.google.protobuf.CodedInputStream input, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws com.google.protobuf.InvalidProtocolBufferException { - return new StatCollectResponse(input, extensionRegistry); - } - }; - - @java.lang.Override - public com.google.protobuf.Parser getParserForType() { - return PARSER; - } - - private int bitField0_; - // required uint64 rowsScanned = 1; - public static final int ROWSSCANNED_FIELD_NUMBER = 1; - private long rowsScanned_; - /** - * required uint64 rowsScanned = 1; - */ - public boolean hasRowsScanned() { - return ((bitField0_ & 0x00000001) == 0x00000001); - } - /** - * required uint64 rowsScanned = 1; - */ - public long getRowsScanned() { - return rowsScanned_; - } - - private void initFields() { - rowsScanned_ = 0L; - } - private byte memoizedIsInitialized = -1; - public final boolean isInitialized() { - byte isInitialized = memoizedIsInitialized; - if (isInitialized != -1) return isInitialized == 1; - - if (!hasRowsScanned()) { - memoizedIsInitialized = 0; - return false; - } - memoizedIsInitialized = 1; - return true; - } - - public void writeTo(com.google.protobuf.CodedOutputStream output) - throws java.io.IOException { - getSerializedSize(); - if (((bitField0_ & 0x00000001) == 0x00000001)) { - output.writeUInt64(1, rowsScanned_); - } - getUnknownFields().writeTo(output); - } - - private int memoizedSerializedSize = -1; - public int getSerializedSize() { - int size = memoizedSerializedSize; - if (size != -1) return size; - - size = 0; - if (((bitField0_ & 0x00000001) == 0x00000001)) { - size += com.google.protobuf.CodedOutputStream - .computeUInt64Size(1, rowsScanned_); - } - size += getUnknownFields().getSerializedSize(); - memoizedSerializedSize = size; - return size; - } - - private static final long serialVersionUID = 0L; - @java.lang.Override - protected java.lang.Object writeReplace() - throws java.io.ObjectStreamException { - return super.writeReplace(); - } - - @java.lang.Override - public boolean equals(final java.lang.Object obj) { - if (obj == this) { - return true; - } - if (!(obj instanceof org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectResponse)) { - return super.equals(obj); - } - org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectResponse other = (org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectResponse) obj; - - boolean result = true; - result = result && (hasRowsScanned() == other.hasRowsScanned()); - if (hasRowsScanned()) { - result = result && (getRowsScanned() - == other.getRowsScanned()); - } - result = result && - getUnknownFields().equals(other.getUnknownFields()); - return result; - } - - private int memoizedHashCode = 0; - @java.lang.Override - public int hashCode() { - if (memoizedHashCode != 0) { - return memoizedHashCode; - } - int hash = 41; - hash = (19 * hash) + getDescriptorForType().hashCode(); - if (hasRowsScanned()) { - hash = (37 * hash) + ROWSSCANNED_FIELD_NUMBER; - hash = (53 * hash) + hashLong(getRowsScanned()); - } - hash = (29 * hash) + getUnknownFields().hashCode(); - memoizedHashCode = hash; - return hash; - } - - public static org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectResponse parseFrom( - com.google.protobuf.ByteString data) - throws com.google.protobuf.InvalidProtocolBufferException { - return PARSER.parseFrom(data); - } - public static org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectResponse parseFrom( - com.google.protobuf.ByteString data, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws com.google.protobuf.InvalidProtocolBufferException { - return PARSER.parseFrom(data, extensionRegistry); - } - public static org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectResponse parseFrom(byte[] data) - throws com.google.protobuf.InvalidProtocolBufferException { - return PARSER.parseFrom(data); - } - public static org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectResponse parseFrom( - byte[] data, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws com.google.protobuf.InvalidProtocolBufferException { - return PARSER.parseFrom(data, extensionRegistry); - } - public static org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectResponse parseFrom(java.io.InputStream input) - throws java.io.IOException { - return PARSER.parseFrom(input); - } - public static org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectResponse parseFrom( - java.io.InputStream input, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws java.io.IOException { - return PARSER.parseFrom(input, extensionRegistry); - } - public static org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectResponse parseDelimitedFrom(java.io.InputStream input) - throws java.io.IOException { - return PARSER.parseDelimitedFrom(input); - } - public static org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectResponse parseDelimitedFrom( - java.io.InputStream input, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws java.io.IOException { - return PARSER.parseDelimitedFrom(input, extensionRegistry); - } - public static org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectResponse parseFrom( - com.google.protobuf.CodedInputStream input) - throws java.io.IOException { - return PARSER.parseFrom(input); - } - public static org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectResponse parseFrom( - com.google.protobuf.CodedInputStream input, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws java.io.IOException { - return PARSER.parseFrom(input, extensionRegistry); - } - - public static Builder newBuilder() { return Builder.create(); } - public Builder newBuilderForType() { return newBuilder(); } - public static Builder newBuilder(org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectResponse prototype) { - return newBuilder().mergeFrom(prototype); - } - public Builder toBuilder() { return newBuilder(this); } - - @java.lang.Override - protected Builder newBuilderForType( - com.google.protobuf.GeneratedMessage.BuilderParent parent) { - Builder builder = new Builder(parent); - return builder; - } - /** - * Protobuf type {@code StatCollectResponse} - */ - public static final class Builder extends - com.google.protobuf.GeneratedMessage.Builder - implements org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectResponseOrBuilder { - public static final com.google.protobuf.Descriptors.Descriptor - getDescriptor() { - return org.apache.phoenix.coprocessor.generated.StatCollectorProtos.internal_static_StatCollectResponse_descriptor; - } - - protected com.google.protobuf.GeneratedMessage.FieldAccessorTable - internalGetFieldAccessorTable() { - return org.apache.phoenix.coprocessor.generated.StatCollectorProtos.internal_static_StatCollectResponse_fieldAccessorTable - .ensureFieldAccessorsInitialized( - org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectResponse.class, org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectResponse.Builder.class); - } - - // Construct using org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectResponse.newBuilder() - private Builder() { - maybeForceBuilderInitialization(); - } - - private Builder( - com.google.protobuf.GeneratedMessage.BuilderParent parent) { - super(parent); - maybeForceBuilderInitialization(); - } - private void maybeForceBuilderInitialization() { - if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) { - } - } - private static Builder create() { - return new Builder(); - } - - public Builder clear() { - super.clear(); - rowsScanned_ = 0L; - bitField0_ = (bitField0_ & ~0x00000001); - return this; - } - - public Builder clone() { - return create().mergeFrom(buildPartial()); - } - - public com.google.protobuf.Descriptors.Descriptor - getDescriptorForType() { - return org.apache.phoenix.coprocessor.generated.StatCollectorProtos.internal_static_StatCollectResponse_descriptor; - } - - public org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectResponse getDefaultInstanceForType() { - return org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectResponse.getDefaultInstance(); - } - - public org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectResponse build() { - org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectResponse result = buildPartial(); - if (!result.isInitialized()) { - throw newUninitializedMessageException(result); - } - return result; - } - - public org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectResponse buildPartial() { - org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectResponse result = new org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectResponse(this); - int from_bitField0_ = bitField0_; - int to_bitField0_ = 0; - if (((from_bitField0_ & 0x00000001) == 0x00000001)) { - to_bitField0_ |= 0x00000001; - } - result.rowsScanned_ = rowsScanned_; - result.bitField0_ = to_bitField0_; - onBuilt(); - return result; - } - - public Builder mergeFrom(com.google.protobuf.Message other) { - if (other instanceof org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectResponse) { - return mergeFrom((org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectResponse)other); - } else { - super.mergeFrom(other); - return this; - } - } - - public Builder mergeFrom(org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectResponse other) { - if (other == org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectResponse.getDefaultInstance()) return this; - if (other.hasRowsScanned()) { - setRowsScanned(other.getRowsScanned()); - } - this.mergeUnknownFields(other.getUnknownFields()); - return this; - } - - public final boolean isInitialized() { - if (!hasRowsScanned()) { - - return false; - } - return true; - } - - public Builder mergeFrom( - com.google.protobuf.CodedInputStream input, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws java.io.IOException { - org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectResponse parsedMessage = null; - try { - parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry); - } catch (com.google.protobuf.InvalidProtocolBufferException e) { - parsedMessage = (org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectResponse) e.getUnfinishedMessage(); - throw e; - } finally { - if (parsedMessage != null) { - mergeFrom(parsedMessage); - } - } - return this; - } - private int bitField0_; - - // required uint64 rowsScanned = 1; - private long rowsScanned_ ; - /** - * required uint64 rowsScanned = 1; - */ - public boolean hasRowsScanned() { - return ((bitField0_ & 0x00000001) == 0x00000001); - } - /** - * required uint64 rowsScanned = 1; - */ - public long getRowsScanned() { - return rowsScanned_; - } - /** - * required uint64 rowsScanned = 1; - */ - public Builder setRowsScanned(long value) { - bitField0_ |= 0x00000001; - rowsScanned_ = value; - onChanged(); - return this; - } - /** - * required uint64 rowsScanned = 1; - */ - public Builder clearRowsScanned() { - bitField0_ = (bitField0_ & ~0x00000001); - rowsScanned_ = 0L; - onChanged(); - return this; - } - - // @@protoc_insertion_point(builder_scope:StatCollectResponse) - } - - static { - defaultInstance = new StatCollectResponse(true); - defaultInstance.initFields(); - } - - // @@protoc_insertion_point(class_scope:StatCollectResponse) - } - - /** - * Protobuf service {@code StatCollectService} - */ - public static abstract class StatCollectService - implements com.google.protobuf.Service { - protected StatCollectService() {} - - public interface Interface { - /** - * rpc collectStat(.StatCollectRequest) returns (.StatCollectResponse); - */ - public abstract void collectStat( - com.google.protobuf.RpcController controller, - org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectRequest request, - com.google.protobuf.RpcCallback done); - - } - - public static com.google.protobuf.Service newReflectiveService( - final Interface impl) { - return new StatCollectService() { - @java.lang.Override - public void collectStat( - com.google.protobuf.RpcController controller, - org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectRequest request, - com.google.protobuf.RpcCallback done) { - impl.collectStat(controller, request, done); - } - - }; - } - - public static com.google.protobuf.BlockingService - newReflectiveBlockingService(final BlockingInterface impl) { - return new com.google.protobuf.BlockingService() { - public final com.google.protobuf.Descriptors.ServiceDescriptor - getDescriptorForType() { - return getDescriptor(); - } - - public final com.google.protobuf.Message callBlockingMethod( - com.google.protobuf.Descriptors.MethodDescriptor method, - com.google.protobuf.RpcController controller, - com.google.protobuf.Message request) - throws com.google.protobuf.ServiceException { - if (method.getService() != getDescriptor()) { - throw new java.lang.IllegalArgumentException( - "Service.callBlockingMethod() given method descriptor for " + - "wrong service type."); - } - switch(method.getIndex()) { - case 0: - return impl.collectStat(controller, (org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectRequest)request); - default: - throw new java.lang.AssertionError("Can't get here."); - } - } - - public final com.google.protobuf.Message - getRequestPrototype( - com.google.protobuf.Descriptors.MethodDescriptor method) { - if (method.getService() != getDescriptor()) { - throw new java.lang.IllegalArgumentException( - "Service.getRequestPrototype() given method " + - "descriptor for wrong service type."); - } - switch(method.getIndex()) { - case 0: - return org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectRequest.getDefaultInstance(); - default: - throw new java.lang.AssertionError("Can't get here."); - } - } - - public final com.google.protobuf.Message - getResponsePrototype( - com.google.protobuf.Descriptors.MethodDescriptor method) { - if (method.getService() != getDescriptor()) { - throw new java.lang.IllegalArgumentException( - "Service.getResponsePrototype() given method " + - "descriptor for wrong service type."); - } - switch(method.getIndex()) { - case 0: - return org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectResponse.getDefaultInstance(); - default: - throw new java.lang.AssertionError("Can't get here."); - } - } - - }; - } - - /** - * rpc collectStat(.StatCollectRequest) returns (.StatCollectResponse); - */ - public abstract void collectStat( - com.google.protobuf.RpcController controller, - org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectRequest request, - com.google.protobuf.RpcCallback done); - - public static final - com.google.protobuf.Descriptors.ServiceDescriptor - getDescriptor() { - return org.apache.phoenix.coprocessor.generated.StatCollectorProtos.getDescriptor().getServices().get(0); - } - public final com.google.protobuf.Descriptors.ServiceDescriptor - getDescriptorForType() { - return getDescriptor(); - } - - public final void callMethod( - com.google.protobuf.Descriptors.MethodDescriptor method, - com.google.protobuf.RpcController controller, - com.google.protobuf.Message request, - com.google.protobuf.RpcCallback< - com.google.protobuf.Message> done) { - if (method.getService() != getDescriptor()) { - throw new java.lang.IllegalArgumentException( - "Service.callMethod() given method descriptor for wrong " + - "service type."); - } - switch(method.getIndex()) { - case 0: - this.collectStat(controller, (org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectRequest)request, - com.google.protobuf.RpcUtil.specializeCallback( - done)); - return; - default: - throw new java.lang.AssertionError("Can't get here."); - } - } - - public final com.google.protobuf.Message - getRequestPrototype( - com.google.protobuf.Descriptors.MethodDescriptor method) { - if (method.getService() != getDescriptor()) { - throw new java.lang.IllegalArgumentException( - "Service.getRequestPrototype() given method " + - "descriptor for wrong service type."); - } - switch(method.getIndex()) { - case 0: - return org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectRequest.getDefaultInstance(); - default: - throw new java.lang.AssertionError("Can't get here."); - } - } - - public final com.google.protobuf.Message - getResponsePrototype( - com.google.protobuf.Descriptors.MethodDescriptor method) { - if (method.getService() != getDescriptor()) { - throw new java.lang.IllegalArgumentException( - "Service.getResponsePrototype() given method " + - "descriptor for wrong service type."); - } - switch(method.getIndex()) { - case 0: - return org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectResponse.getDefaultInstance(); - default: - throw new java.lang.AssertionError("Can't get here."); - } - } - - public static Stub newStub( - com.google.protobuf.RpcChannel channel) { - return new Stub(channel); - } - - public static final class Stub extends org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectService implements Interface { - private Stub(com.google.protobuf.RpcChannel channel) { - this.channel = channel; - } - - private final com.google.protobuf.RpcChannel channel; - - public com.google.protobuf.RpcChannel getChannel() { - return channel; - } - - public void collectStat( - com.google.protobuf.RpcController controller, - org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectRequest request, - com.google.protobuf.RpcCallback done) { - channel.callMethod( - getDescriptor().getMethods().get(0), - controller, - request, - org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectResponse.getDefaultInstance(), - com.google.protobuf.RpcUtil.generalizeCallback( - done, - org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectResponse.class, - org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectResponse.getDefaultInstance())); - } - } - - public static BlockingInterface newBlockingStub( - com.google.protobuf.BlockingRpcChannel channel) { - return new BlockingStub(channel); - } - - public interface BlockingInterface { - public org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectResponse collectStat( - com.google.protobuf.RpcController controller, - org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectRequest request) - throws com.google.protobuf.ServiceException; - } - - private static final class BlockingStub implements BlockingInterface { - private BlockingStub(com.google.protobuf.BlockingRpcChannel channel) { - this.channel = channel; - } - - private final com.google.protobuf.BlockingRpcChannel channel; - - public org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectResponse collectStat( - com.google.protobuf.RpcController controller, - org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectRequest request) - throws com.google.protobuf.ServiceException { - return (org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectResponse) channel.callBlockingMethod( - getDescriptor().getMethods().get(0), - controller, - request, - org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectResponse.getDefaultInstance()); - } - - } - - // @@protoc_insertion_point(class_scope:StatCollectService) - } - - private static com.google.protobuf.Descriptors.Descriptor - internal_static_StatCollectRequest_descriptor; - private static - com.google.protobuf.GeneratedMessage.FieldAccessorTable - internal_static_StatCollectRequest_fieldAccessorTable; - private static com.google.protobuf.Descriptors.Descriptor - internal_static_StatCollectResponse_descriptor; - private static - com.google.protobuf.GeneratedMessage.FieldAccessorTable - internal_static_StatCollectResponse_fieldAccessorTable; - - public static com.google.protobuf.Descriptors.FileDescriptor - getDescriptor() { - return descriptor; - } - private static com.google.protobuf.Descriptors.FileDescriptor - descriptor; - static { - java.lang.String[] descriptorData = { - "\n\027StatisticsCollect.proto\"7\n\022StatCollect" + - "Request\022\020\n\010startRow\030\001 \001(\014\022\017\n\007stopRow\030\002 \001" + - "(\014\"*\n\023StatCollectResponse\022\023\n\013rowsScanned" + - "\030\001 \002(\0042N\n\022StatCollectService\0228\n\013collectS" + - "tat\022\023.StatCollectRequest\032\024.StatCollectRe" + - "sponseBG\n(org.apache.phoenix.coprocessor" + - ".generatedB\023StatCollectorProtosH\001\210\001\001\240\001\001" - }; - com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner = - new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() { - public com.google.protobuf.ExtensionRegistry assignDescriptors( - com.google.protobuf.Descriptors.FileDescriptor root) { - descriptor = root; - internal_static_StatCollectRequest_descriptor = - getDescriptor().getMessageTypes().get(0); - internal_static_StatCollectRequest_fieldAccessorTable = new - com.google.protobuf.GeneratedMessage.FieldAccessorTable( - internal_static_StatCollectRequest_descriptor, - new java.lang.String[] { "StartRow", "StopRow", }); - internal_static_StatCollectResponse_descriptor = - getDescriptor().getMessageTypes().get(1); - internal_static_StatCollectResponse_fieldAccessorTable = new - com.google.protobuf.GeneratedMessage.FieldAccessorTable( - internal_static_StatCollectResponse_descriptor, - new java.lang.String[] { "RowsScanned", }); - return null; - } - }; - com.google.protobuf.Descriptors.FileDescriptor - .internalBuildGeneratedFileFrom(descriptorData, - new com.google.protobuf.Descriptors.FileDescriptor[] { - }, assigner); - } - - // @@protoc_insertion_point(outer_class_scope) -} diff --git a/phoenix-core/src/main/java/org/apache/phoenix/execute/ScanPlan.java b/phoenix-core/src/main/java/org/apache/phoenix/execute/ScanPlan.java index 816cd1ba24b..efb00a8fff6 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/execute/ScanPlan.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/execute/ScanPlan.java @@ -122,7 +122,7 @@ private static boolean isSerial(StatementContext context, } else { // Region size estimated based on total number of bytes divided by number of regions long totByteSize = gpsInfo.getByteCount(); - estRegionSize = totByteSize / (gpsInfo.getGuidePosts().size()+1); + estRegionSize = totByteSize / (gpsInfo.getGuidePostsCount()+1); } // TODO: configurable number of bytes? boolean isSerial = (perScanLimit * estRowSize < estRegionSize); diff --git a/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java b/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java index 519b38f171c..1a5cdc55cfb 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java @@ -22,6 +22,11 @@ import static org.apache.phoenix.monitoring.GlobalClientMetrics.GLOBAL_QUERY_TIMEOUT_COUNTER; import static org.apache.phoenix.util.ByteUtil.EMPTY_BYTE_ARRAY; +import java.io.ByteArrayInputStream; +import java.io.DataInput; +import java.io.DataInputStream; +import java.io.EOFException; +import java.io.IOException; import java.sql.SQLException; import java.util.ArrayList; import java.util.Collections; @@ -46,6 +51,7 @@ import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.filter.FirstKeyOnlyFilter; import org.apache.hadoop.hbase.filter.PageFilter; +import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Pair; import org.apache.phoenix.compile.QueryPlan; @@ -75,7 +81,10 @@ import org.apache.phoenix.schema.TableRef; import org.apache.phoenix.schema.stats.GuidePostsInfo; import org.apache.phoenix.schema.stats.PTableStats; +import org.apache.phoenix.util.ByteUtil; +import org.apache.phoenix.util.CodecUtils; import org.apache.phoenix.util.LogUtil; +import org.apache.phoenix.util.PrefixByteDecoder; import org.apache.phoenix.util.SQLCloseables; import org.apache.phoenix.util.ScanUtil; import org.apache.phoenix.util.SchemaUtil; @@ -356,22 +365,18 @@ private static int getIndexContainingExclusive(List boundaries, byte[] e guideIndex = (guideIndex < 0 ? -(guideIndex + 1) : guideIndex); return guideIndex; } - - private List getGuidePosts(Set whereConditions) { + + private GuidePostsInfo getGuidePosts(Set whereConditions) { /* - * Don't use guide posts if: - * 1) We're doing a point lookup, as HBase is fast enough at those - * to not need them to be further parallelized. TODO: pref test to verify - * 2) We're collecting stats, as in this case we need to scan entire - * regions worth of data to track where to put the guide posts. + * Don't use guide posts if: 1) We're doing a point lookup, as HBase is fast enough at those to not need them to + * be further parallelized. TODO: pref test to verify 2) We're collecting stats, as in this case we need to scan + * entire regions worth of data to track where to put the guide posts. */ - if (!useStats()) { - return Collections.emptyList(); - } - - List gps = null; + if (!useStats()) { return GuidePostsInfo.EMPTY_GUIDEPOST; } + + GuidePostsInfo gps = null; PTable table = getTable(); - Map guidePostMap = tableStats.getGuidePosts(); + Map guidePostMap = tableStats.getGuidePosts(); byte[] defaultCF = SchemaUtil.getEmptyColumnFamily(getTable()); if (table.getColumnFamilies().isEmpty()) { // For sure we can get the defaultCF from the table @@ -379,37 +384,35 @@ private List getGuidePosts(Set whereConditions) { } else { byte[] familyInWhere = null; if (!whereConditions.isEmpty()) { - if (whereConditions.contains(defaultCF)) { - gps = getDefaultFamilyGuidePosts(guidePostMap, defaultCF); - } else { - familyInWhere = whereConditions.iterator().next(); - if(familyInWhere != null) { - GuidePostsInfo guidePostsInfo = guidePostMap.get(familyInWhere); - if (guidePostsInfo != null) { - gps = guidePostsInfo.getGuidePosts(); - } else { - // As there are no guideposts collected for the where family we go with the default CF + if (whereConditions.contains(defaultCF)) { gps = getDefaultFamilyGuidePosts(guidePostMap, defaultCF); - } + } else { + familyInWhere = whereConditions.iterator().next(); + if (familyInWhere != null) { + GuidePostsInfo guidePostsInfo = guidePostMap.get(familyInWhere); + if (guidePostsInfo != null) { + gps = guidePostsInfo; + } else { + // As there are no guideposts collected for the where family we go with the default CF + gps = getDefaultFamilyGuidePosts(guidePostMap, defaultCF); + } + } } - } } else { - gps = getDefaultFamilyGuidePosts(guidePostMap, defaultCF); + gps = getDefaultFamilyGuidePosts(guidePostMap, defaultCF); } } - if (gps == null) { - return Collections.emptyList(); - } + if (gps == null) { return GuidePostsInfo.EMPTY_GUIDEPOST; } return gps; } - private List getDefaultFamilyGuidePosts(Map guidePostMap, byte[] defaultCF) { - if (guidePostMap.get(defaultCF) != null) { - return guidePostMap.get(defaultCF).getGuidePosts(); - } - return null; + private GuidePostsInfo getDefaultFamilyGuidePosts(Map guidePostMap, byte[] defaultCF) { + if (guidePostMap.get(defaultCF) != null) { + return guidePostMap.get(defaultCF); + } + return null; } - + private static String toString(List gps) { StringBuilder buf = new StringBuilder(gps.size() * 100); buf.append("["); @@ -461,9 +464,9 @@ private List> getParallelScans(byte[] startKey, byte[] stopKey) throw for(Pair where : context.getWhereConditionColumns()) { whereConditions.add(where.getFirst()); } - List gps = getGuidePosts(whereConditions); + GuidePostsInfo gps = getGuidePosts(whereConditions); if (logger.isDebugEnabled()) { - logger.debug("Guideposts: " + toString(gps)); + logger.debug("Guideposts: " + gps); } boolean traverseAllRegions = isSalted || isLocalIndex; if (!traverseAllRegions) { @@ -490,15 +493,37 @@ private List> getParallelScans(byte[] startKey, byte[] stopKey) throw } List> parallelScans = Lists.newArrayListWithExpectedSize(stopIndex - regionIndex + 1); - byte[] currentKey = startKey; - int guideIndex = currentKey.length == 0 ? 0 : getIndexContainingInclusive(gps, currentKey); - int gpsSize = gps.size(); + ImmutableBytesWritable currentKey = new ImmutableBytesWritable(startKey, 0, startKey.length); + + int gpsSize = gps.getGuidePostsCount(); int estGuidepostsPerRegion = gpsSize == 0 ? 1 : gpsSize / regionLocations.size() + 1; int keyOffset = 0; + ImmutableBytesWritable currentGuidePost = ByteUtil.EMPTY_IMMUTABLE_BYTE_ARRAY; List scans = Lists.newArrayListWithExpectedSize(estGuidepostsPerRegion); + ImmutableBytesWritable guidePosts = gps.getGuidePosts(); + ByteArrayInputStream stream = null; + DataInput input = null; + PrefixByteDecoder decoder = null; + int guideIndex = 0; + if (gpsSize > 0) { + stream = new ByteArrayInputStream(guidePosts.get(), guidePosts.getOffset(), guidePosts.getLength()); + input = new DataInputStream(stream); + if (gps.getMaxLength() > 0) { + decoder = new PrefixByteDecoder(gps.getMaxLength()); + } else { + decoder = new PrefixByteDecoder(); + } + try { + currentGuidePost = CodecUtils.decode(decoder, input); + while (currentKey.getLength() != 0 && Bytes.compareTo(currentKey.get(), currentGuidePost.get()) <= 0) { + currentGuidePost = CodecUtils.decode(decoder, input); + guideIndex++; + } + } catch (EOFException e) {} + } // Merge bisect with guideposts for all but the last region while (regionIndex <= stopIndex) { - byte[] currentGuidePost, endKey, endRegionKey = EMPTY_BYTE_ARRAY; + byte[] endKey, endRegionKey = EMPTY_BYTE_ARRAY; if (regionIndex == stopIndex) { endKey = stopKey; } else { @@ -510,14 +535,18 @@ private List> getParallelScans(byte[] startKey, byte[] stopKey) throw endRegionKey = regionInfo.getEndKey(); keyOffset = ScanUtil.getRowKeyOffset(regionInfo.getStartKey(), endRegionKey); } - while (guideIndex < gpsSize - && (Bytes.compareTo(currentGuidePost = gps.get(guideIndex), endKey) <= 0 || endKey.length == 0)) { - Scan newScan = scanRanges.intersectScan(scan, currentKey, currentGuidePost, keyOffset, false); - scans = addNewScan(parallelScans, scans, newScan, currentGuidePost, false, regionLocation); - currentKey = currentGuidePost; - guideIndex++; - } - Scan newScan = scanRanges.intersectScan(scan, currentKey, endKey, keyOffset, true); + try { + while (guideIndex < gpsSize + && (Bytes.compareTo(currentGuidePost.get(), + endKey) <= 0 || endKey.length == 0)) { + Scan newScan = scanRanges.intersectScan(scan, currentKey, currentGuidePost, keyOffset, false); + scans = addNewScan(parallelScans, scans, newScan, currentGuidePost.get(), false, regionLocation); + currentKey = new ImmutableBytesWritable(currentGuidePost.copyBytes()); + currentGuidePost = CodecUtils.decode(decoder, input); + guideIndex++; + } + } catch (EOFException e) {} + Scan newScan = scanRanges.intersectScan(scan, currentKey, new ImmutableBytesWritable(endKey,0,endKey.length), keyOffset, true); if (isLocalIndex) { if (newScan != null) { newScan.setAttribute(EXPECTED_UPPER_REGION_KEY, endRegionKey); @@ -526,15 +555,21 @@ private List> getParallelScans(byte[] startKey, byte[] stopKey) throw } } scans = addNewScan(parallelScans, scans, newScan, endKey, true, regionLocation); - currentKey = endKey; + currentKey = new ImmutableBytesWritable(endKey, 0, endKey.length); regionIndex++; } if (!scans.isEmpty()) { // Add any remaining scans parallelScans.add(scans); } + if (stream != null) { + try { + stream.close(); + } catch (IOException e) {} + } return parallelScans; } + public static List reverseIfNecessary(List list, boolean reverse) { if (!reverse) { return list; @@ -805,8 +840,9 @@ public void explain(List planSteps) { explain(buf.toString(),planSteps); } - @Override - public String toString() { - return "ResultIterators [name=" + getName() + ",id=" + scanId + ",scans=" + scans + "]"; - } + @Override + public String toString() { + return "ResultIterators [name=" + getName() + ",id=" + scanId + ",scans=" + scans + "]"; + } + } \ No newline at end of file diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java index 1805d94bffd..e1e76e5179a 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java @@ -1003,26 +1003,16 @@ public static PTable createFromProto(PTableProtos.PTable table) { boolean isImmutableRows = table.getIsImmutableRows(); SortedMap tableGuidePosts = new TreeMap(Bytes.BYTES_COMPARATOR); for (PTableProtos.PTableStats pTableStatsProto : table.getGuidePostsList()) { - List value = Lists.newArrayListWithExpectedSize(pTableStatsProto.getValuesCount()); - for (int j = 0; j < pTableStatsProto.getValuesCount(); j++) { - value.add(pTableStatsProto.getValues(j).toByteArray()); - } - // No op - pTableStatsProto.getGuidePostsByteCount(); - value = Lists.newArrayListWithExpectedSize(pTableStatsProto.getValuesCount()); PGuidePosts pGuidePosts = pTableStatsProto.getPGuidePosts(); - for(int j = 0; j < pGuidePosts.getGuidePostsCount(); j++) { - value.add(pGuidePosts.getGuidePosts(j).toByteArray()); - } long guidePostsByteCount = pGuidePosts.getByteCount(); long rowCount = pGuidePosts.getRowCount(); - // TODO : Not exposing MIN/MAX key outside to client - GuidePostsInfo info = - new GuidePostsInfo(guidePostsByteCount, value, rowCount); + int maxLength = pGuidePosts.getMaxLength(); + int guidePostsCount = pGuidePosts.getGuidePostsCount(); + GuidePostsInfo info = new GuidePostsInfo(guidePostsByteCount, + new ImmutableBytesWritable(pGuidePosts.getGuidePosts().toByteArray()), rowCount, maxLength, guidePostsCount); tableGuidePosts.put(pTableStatsProto.getKey().toByteArray(), info); - } + } PTableStats stats = new PTableStatsImpl(tableGuidePosts, table.getStatsTimeStamp()); - PName dataTableName = null; if (table.hasDataTableNameBytes()) { dataTableName = PNameFactory.newName(table.getDataTableNameBytes().toByteArray()); @@ -1121,16 +1111,16 @@ public static PTableProtos.PTable toProto(PTable table) { for (Map.Entry entry : table.getTableStats().getGuidePosts().entrySet()) { PTableProtos.PTableStats.Builder statsBuilder = PTableProtos.PTableStats.newBuilder(); statsBuilder.setKey(ByteStringer.wrap(entry.getKey())); - for (byte[] stat : entry.getValue().getGuidePosts()) { - statsBuilder.addValues(ByteStringer.wrap(stat)); - } + statsBuilder.setGuidePosts(ByteStringer.wrap(entry.getValue().getGuidePosts().get())); statsBuilder.setGuidePostsByteCount(entry.getValue().getByteCount()); + statsBuilder.setMaxLength(entry.getValue().getMaxLength()); + statsBuilder.setGuidePostsCount(entry.getValue().getGuidePostsCount()); PGuidePostsProtos.PGuidePosts.Builder guidePstsBuilder = PGuidePostsProtos.PGuidePosts.newBuilder(); - for (byte[] stat : entry.getValue().getGuidePosts()) { - guidePstsBuilder.addGuidePosts(ByteStringer.wrap(stat)); - } + guidePstsBuilder.setGuidePosts(ByteStringer.wrap(entry.getValue().getGuidePosts().get())); guidePstsBuilder.setByteCount(entry.getValue().getByteCount()); guidePstsBuilder.setRowCount(entry.getValue().getRowCount()); + guidePstsBuilder.setMaxLength(entry.getValue().getMaxLength()); + guidePstsBuilder.setGuidePostsCount(entry.getValue().getGuidePostsCount()); statsBuilder.setPGuidePosts(guidePstsBuilder); builder.addGuidePosts(statsBuilder.build()); } diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsInfo.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsInfo.java index 0f1dbeb490f..00fb3499385 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsInfo.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsInfo.java @@ -17,14 +17,14 @@ */ package org.apache.phoenix.schema.stats; -import java.util.ArrayList; -import java.util.List; +import java.io.DataOutputStream; +import java.io.IOException; -import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import org.apache.hadoop.hbase.util.Bytes; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.Lists; +import org.apache.phoenix.util.ByteUtil; +import org.apache.phoenix.util.PrefixByteEncoder; +import org.apache.phoenix.util.TrustedByteArrayOutputStream; /** * A class that holds the guidePosts of a region and also allows combining the * guidePosts of different regions when the GuidePostsInfo is formed for a table. @@ -34,7 +34,7 @@ public class GuidePostsInfo { /** * the total number of guidePosts for the table combining all the guidePosts per region per cf. */ - private List guidePosts; + private ImmutableBytesWritable guidePosts; /** * The bytecount that is flattened across the total number of guide posts. */ @@ -45,7 +45,24 @@ public class GuidePostsInfo { */ private long rowCount = 0; - private long keyByteSize; // Total number of bytes in keys stored in guidePosts + /** + * Maximum length of a guidePost collected + */ + private int maxLength; + + public final static GuidePostsInfo EMPTY_GUIDEPOST = new GuidePostsInfo(0, + new ImmutableBytesWritable(ByteUtil.EMPTY_BYTE_ARRAY), 0, 0, 0); + + public int getMaxLength() { + return maxLength; + } + + private TrustedByteArrayOutputStream stream; + private PrefixByteEncoder encoder; + private DataOutputStream output; + private byte[] lastRow; + private int guidePostsCount; + private boolean isStreamInitialized; /** * Constructor that creates GuidePostsInfo per region @@ -53,22 +70,21 @@ public class GuidePostsInfo { * @param guidePosts * @param rowCount */ - public GuidePostsInfo(long byteCount, List guidePosts, long rowCount) { - this.guidePosts = ImmutableList.copyOf(guidePosts); - int size = 0; - for (byte[] key : guidePosts) { - size += key.length; - } - this.keyByteSize = size; + public GuidePostsInfo(long byteCount, ImmutableBytesWritable guidePosts, long rowCount, int maxLength, int guidePostsCount) { + this.guidePosts = guidePosts; + this.maxLength = maxLength; this.byteCount = byteCount; this.rowCount = rowCount; + this.guidePostsCount = guidePostsCount; + lastRow = ByteUtil.EMPTY_BYTE_ARRAY; } + public long getByteCount() { return byteCount; } - public List getGuidePosts() { + public ImmutableBytesWritable getGuidePosts() { return guidePosts; } @@ -79,67 +95,72 @@ public long getRowCount() { public void incrementRowCount() { this.rowCount++; } - - /** - * Combines the GuidePosts per region into one. - * @param oldInfo - */ - public void combine(GuidePostsInfo oldInfo) { - if (!oldInfo.getGuidePosts().isEmpty()) { - byte[] newFirstKey = oldInfo.getGuidePosts().get(0); - byte[] existingLastKey; - if (!this.getGuidePosts().isEmpty()) { - existingLastKey = this.getGuidePosts().get(this.getGuidePosts().size() - 1); - } else { - existingLastKey = HConstants.EMPTY_BYTE_ARRAY; - } - int size = oldInfo.getGuidePosts().size(); - // If the existing guidePosts is lesser than the new RegionInfo that we are combining - // then add the new Region info to the end of the current GuidePosts. - // If the new region info is smaller than the existing guideposts then add the existing - // guide posts after the new guideposts. - List newTotalGuidePosts = new ArrayList(this.getGuidePosts().size() + size); - if (Bytes.compareTo(existingLastKey, newFirstKey) <= 0) { - newTotalGuidePosts.addAll(this.getGuidePosts()); - newTotalGuidePosts.addAll(oldInfo.getGuidePosts()); - } else { - newTotalGuidePosts.addAll(oldInfo.getGuidePosts()); - newTotalGuidePosts.addAll(this.getGuidePosts()); - } - this.guidePosts = ImmutableList.copyOf(newTotalGuidePosts); - } - this.byteCount += oldInfo.getByteCount(); - this.keyByteSize += oldInfo.keyByteSize; - this.rowCount += oldInfo.getRowCount(); - } + public int getGuidePostsCount() { + return guidePostsCount; + } + /** * The guide posts, rowCount and byteCount are accumulated every time a guidePosts depth is * reached while collecting stats. * @param row * @param byteCount * @return + * @throws IOException */ - public boolean addGuidePost(byte[] row, long byteCount, long rowCount) { - if (guidePosts.isEmpty() || Bytes.compareTo(row, guidePosts.get(guidePosts.size() - 1)) > 0) { - List newGuidePosts = Lists.newArrayListWithExpectedSize(this.getGuidePosts().size() + 1); - newGuidePosts.addAll(guidePosts); - newGuidePosts.add(row); - this.guidePosts = ImmutableList.copyOf(newGuidePosts); - this.byteCount += byteCount; - this.keyByteSize += row.length; - this.rowCount+=rowCount; - return true; + public boolean encodeAndCollectGuidePost(byte[] row, long byteCount, long rowCount) { + if (row.length != 0 && Bytes.compareTo(lastRow, row) < 0) { + try { + if(!isStreamInitialized){ + stream = new TrustedByteArrayOutputStream(guidePosts.getLength()); + output = new DataOutputStream(stream); + stream.write(ByteUtil.copyKeyBytesIfNecessary(guidePosts)); + encoder = new PrefixByteEncoder(); + isStreamInitialized=true; + } + encoder.encode(output, row, 0, row.length); + this.byteCount += byteCount; + this.guidePostsCount++; + this.maxLength = encoder.getMaxLength(); + this.rowCount += rowCount; + lastRow = row; + return true; + } catch (IOException e) { + return false; + } } return false; } - - public boolean addGuidePost(byte[] row) { - return addGuidePost(row, 0, 0); + + public boolean encodeAndCollectGuidePost(byte[] row){ + return encodeAndCollectGuidePost(row, 0, 0); } - public boolean addGuidePost(byte[] row, long byteCount) { - return addGuidePost(row, byteCount, 0); + public boolean encodeAndCollectGuidePost(byte[] row, long byteCount){ + return encodeAndCollectGuidePost(row, byteCount, 0); } + public void close() { + if (stream != null) { + try { + stream.close(); + } catch (IOException e) { + //exception can be ignored + } + } + } + + /* + * This needs to be collect once all stats are encoded and stream buffer needs to be copied for retrieval before close() + */ + public void updateGuidePosts() { + if (stream != null) { + this.guidePosts.set(Bytes.copy(stream.getBuffer(), 0, stream.size())); + } + } + + @Override + public String toString() { + return Bytes.toString(this.guidePosts.get()); + } } \ No newline at end of file diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/PTableStatsImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/PTableStatsImpl.java index dc70e8663ac..c6b8d921918 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/PTableStatsImpl.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/PTableStatsImpl.java @@ -17,13 +17,20 @@ */ package org.apache.phoenix.schema.stats; -import java.util.List; +import java.io.ByteArrayInputStream; +import java.io.DataInput; +import java.io.DataInputStream; +import java.io.EOFException; +import java.io.IOException; import java.util.Map; import java.util.SortedMap; import java.util.TreeMap; +import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import org.apache.hadoop.hbase.util.Bytes; import org.apache.phoenix.coprocessor.MetaDataProtocol; +import org.apache.phoenix.util.CodecUtils; +import org.apache.phoenix.util.PrefixByteDecoder; import org.apache.phoenix.util.SizedUtil; import com.sun.istack.NotNull; @@ -47,12 +54,10 @@ public PTableStatsImpl(@NotNull SortedMap guidePosts, lo for (Map.Entry entry : guidePosts.entrySet()) { byte[] cf = entry.getKey(); estimatedSize += SizedUtil.ARRAY_SIZE + cf.length; - List keys = entry.getValue().getGuidePosts(); - estimatedSize += SizedUtil.sizeOfArrayList(keys.size()); - for (byte[] key : keys) { - estimatedSize += SizedUtil.ARRAY_SIZE + key.length; - } + estimatedSize += entry.getValue().getGuidePosts().getLength(); estimatedSize += SizedUtil.LONG_SIZE; + estimatedSize += SizedUtil.INT_SIZE; + estimatedSize += SizedUtil.INT_SIZE; } this.estimatedSize = estimatedSize; } @@ -65,19 +70,45 @@ public SortedMap getGuidePosts() { @Override public String toString() { StringBuilder buf = new StringBuilder(); + buf.append("PTableStats ["); for (Map.Entry entry : guidePosts.entrySet()) { buf.append(Bytes.toStringBinary(entry.getKey())); buf.append(":("); - List keys = entry.getValue().getGuidePosts(); - if (!keys.isEmpty()) { - for (byte[] key : keys) { - buf.append(Bytes.toStringBinary(key)); - buf.append(","); + ImmutableBytesWritable keys = entry.getValue().getGuidePosts(); + ByteArrayInputStream stream = new ByteArrayInputStream(keys.get(), keys.getOffset(), keys.getLength()); + try { + if (keys.getLength() != 0) { + DataInput input = new DataInputStream(stream); + PrefixByteDecoder decoder; + if (entry.getValue().getMaxLength() > 0) { + decoder = new PrefixByteDecoder(entry.getValue().getMaxLength()); + } else { + decoder = new PrefixByteDecoder(); + } + try { + while (true) { + ImmutableBytesWritable ptr = CodecUtils.decode(decoder, input); + buf.append(Bytes.toStringBinary(ptr.get())); + buf.append(","); + } + } catch (EOFException e) { // Ignore as this signifies we're done + + } finally { + try { + stream.close(); + } catch (IOException e) {} + } + buf.setLength(buf.length() - 1); + } + buf.append(")"); + } finally { + if (stream != null) { + try { + stream.close(); + } catch (IOException e) {} } - buf.setLength(buf.length()-1); } - buf.append(")"); } buf.append("]"); return buf.toString(); diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollector.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollector.java index fc8d8bddb4a..e51e2bdca4d 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollector.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollector.java @@ -19,7 +19,6 @@ import java.io.IOException; import java.util.ArrayList; -import java.util.Collections; import java.util.List; import java.util.Map; @@ -99,7 +98,7 @@ public StatisticsCollector(RegionCoprocessorEnvironment env, String tableName, l // in a compaction we know the one family ahead of time if (family != null) { ImmutableBytesPtr cfKey = new ImmutableBytesPtr(family); - cachedGps = new Pair(0l, new GuidePostsInfo(0, Collections. emptyList(), 0l)); + cachedGps = new Pair(0l, new GuidePostsInfo(0, new ImmutableBytesWritable(ByteUtil.EMPTY_BYTE_ARRAY), 0l, 0, 0)); guidePostsMap.put(cfKey, cachedGps); } } @@ -110,6 +109,12 @@ public long getMaxTimeStamp() { public void close() throws IOException { this.statsTable.close(); + for (ImmutableBytesPtr ptr : guidePostsMap.keySet()) { + GuidePostsInfo guidePostInfo = guidePostsMap.get(ptr).getSecond(); + if (guidePostInfo != null) { + guidePostInfo.close(); + } + } } public void updateStatistic(Region region) { @@ -175,7 +180,7 @@ public void collectStatistics(final List results) { gps = guidePostsMap.get(cfKey); if (gps == null) { gps = new Pair(0l, - new GuidePostsInfo(0, Collections. emptyList(), 0l)); + new GuidePostsInfo(0,new ImmutableBytesWritable(ByteUtil.EMPTY_BYTE_ARRAY), 0l, 0, 0)); guidePostsMap.put(cfKey, gps); } if (famMap.get(cfKey) == null) { @@ -191,7 +196,7 @@ public void collectStatistics(final List results) { if (byteCount >= guidepostDepth) { byte[] row = ByteUtil.copyKeyBytesIfNecessary( new ImmutableBytesWritable(kv.getRowArray(), kv.getRowOffset(), kv.getRowLength())); - if (gps.getSecond().addGuidePost(row, byteCount)) { + if (gps.getSecond().encodeAndCollectGuidePost(row, byteCount)) { gps.setFirst(0l); } } @@ -225,21 +230,19 @@ public void clear() { maxTimeStamp = MetaDataProtocol.MIN_TABLE_TIMESTAMP; } - public void addGuidePost(ImmutableBytesPtr cfKey, GuidePostsInfo info, long byteSize, long timestamp, - byte[] minKey) { - Pair newInfo = new Pair(byteSize, info); - Pair oldInfo = guidePostsMap.put(cfKey, newInfo); - if (oldInfo != null) { - info.combine(oldInfo.getSecond()); - newInfo.setFirst(oldInfo.getFirst() + newInfo.getFirst()); - } - maxTimeStamp = Math.max(maxTimeStamp, timestamp); - } - public GuidePostsInfo getGuidePosts(ImmutableBytesPtr fam) { Pair pair = guidePostsMap.get(fam); if (pair != null) { return pair.getSecond(); } return null; } + public void updateGuidePosts() { + for (ImmutableBytesPtr cf : guidePostsMap.keySet()) { + GuidePostsInfo guidePostInfo = guidePostsMap.get(cf).getSecond(); + if (guidePostInfo != null) { + guidePostInfo.updateGuidePosts(); + } + } + } + } diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsScanner.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsScanner.java index d6f0bf13571..ffce33674b3 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsScanner.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsScanner.java @@ -93,6 +93,7 @@ protected void updateStat(final List results) { @Override public void close() throws IOException { boolean async = config.getBoolean(COMMIT_STATS_ASYNC, DEFAULT_COMMIT_STATS_ASYNC); + tracker.updateGuidePosts(); StatisticsCollectionRunTracker collectionTracker = StatisticsCollectionRunTracker.getInstance(config); StatisticsScannerCallable callable = new StatisticsScannerCallable(); if (!async) { @@ -134,7 +135,8 @@ public Void call() throws IOException { } finally { try { collectionTracker.removeCompactingRegion(regionInfo); - stats.close(); + stats.close();// close the writer + tracker.close();// close the tracker } catch (IOException e) { if (toThrow == null) toThrow = e; LOG.error("Error while closing the stats table", e); diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsUtil.java index 16f249c6ecd..5ea7462d140 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsUtil.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsUtil.java @@ -21,7 +21,6 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collections; import java.util.List; import java.util.TreeMap; @@ -55,10 +54,14 @@ private StatisticsUtil() { /** Number of parts in our complex key */ protected static final int NUM_KEY_PARTS = 3; - + public static byte[] getRowKey(byte[] table, ImmutableBytesPtr fam, byte[] guidePostStartKey) { + return getRowKey(table, fam, new ImmutableBytesWritable(guidePostStartKey,0,guidePostStartKey.length)); + } + + public static byte[] getRowKey(byte[] table, ImmutableBytesPtr fam, ImmutableBytesWritable guidePostStartKey) { // always starts with the source table - byte[] rowKey = new byte[table.length + fam.getLength() + guidePostStartKey.length + 2]; + byte[] rowKey = new byte[table.length + fam.getLength() + guidePostStartKey.getLength() + 2]; int offset = 0; System.arraycopy(table, 0, rowKey, offset, table.length); offset += table.length; @@ -66,7 +69,7 @@ public static byte[] getRowKey(byte[] table, ImmutableBytesPtr fam, byte[] guide System.arraycopy(fam.get(), fam.getOffset(), rowKey, offset, fam.getLength()); offset += fam.getLength(); rowKey[offset++] = QueryConstants.SEPARATOR_BYTE; // assumes stats table columns not DESC - System.arraycopy(guidePostStartKey, 0, rowKey, offset, guidePostStartKey.length); + System.arraycopy(guidePostStartKey.get(), 0, rowKey, offset, guidePostStartKey.getLength()); return rowKey; } @@ -170,16 +173,29 @@ public static PTableStats readStatistics(HTableInterface statsHTable, byte[] tab byte[] newGPStartKey = getGuidePostsInfoFromRowKey(tableNameBytes, cfName, result.getRow()); GuidePostsInfo guidePosts = guidePostsPerCf.get(cfName); if (guidePosts == null) { - guidePosts = new GuidePostsInfo(0l, Collections. emptyList(), 0l); + guidePosts = new GuidePostsInfo(0l, new ImmutableBytesWritable(ByteUtil.EMPTY_BYTE_ARRAY), 0l, + 0, 0); guidePostsPerCf.put(cfName, guidePosts); } - guidePosts.addGuidePost(newGPStartKey, byteCount, rowCount); + guidePosts.encodeAndCollectGuidePost(newGPStartKey, byteCount, rowCount); + } + } + for (byte[] cf : guidePostsPerCf.keySet()) { + GuidePostsInfo guidePostInfo = guidePostsPerCf.get(cf); + if (guidePostInfo != null) { + guidePostInfo.updateGuidePosts(); } } } finally { if (scanner != null) { scanner.close(); } + for (byte[] cf : guidePostsPerCf.keySet()) { + GuidePostsInfo guidePostInfo = guidePostsPerCf.get(cf); + if (guidePostInfo != null) { + guidePostInfo.close(); + } + } } if (!guidePostsPerCf.isEmpty()) { return new PTableStatsImpl(guidePostsPerCf, timeStamp); } return PTableStats.EMPTY_STATS; diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsWriter.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsWriter.java index b8bd06450b2..1dcfb3df265 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsWriter.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsWriter.java @@ -17,7 +17,11 @@ */ package org.apache.phoenix.schema.stats; +import java.io.ByteArrayInputStream; import java.io.Closeable; +import java.io.DataInput; +import java.io.DataInputStream; +import java.io.EOFException; import java.io.IOException; import java.sql.Date; import java.util.List; @@ -45,6 +49,7 @@ import org.apache.phoenix.schema.types.PDate; import org.apache.phoenix.schema.types.PLong; import org.apache.phoenix.util.ByteUtil; +import org.apache.phoenix.util.PrefixByteDecoder; import org.apache.phoenix.util.ServerUtil; import org.apache.phoenix.util.TimeKeeper; @@ -103,6 +108,7 @@ private StatisticsWriter(HTableInterface statsReaderTable, HTableInterface stats @Override public void close() throws IOException { statsWriterTable.close(); + statsReaderTable.close(); } /** @@ -134,21 +140,36 @@ public void addStats(StatisticsCollector tracker, ImmutableBytesPtr cfKey, List< GuidePostsInfo gps = tracker.getGuidePosts(cfKey); if (gps != null) { boolean rowColumnAdded = false; - for (byte[] gp : gps.getGuidePosts()) { - byte[] prefix = StatisticsUtil.getRowKey(tableName, cfKey, gp); - Put put = new Put(prefix); - if (!rowColumnAdded) { - put.add(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES, PhoenixDatabaseMetaData.GUIDE_POSTS_WIDTH_BYTES, - timeStamp, PLong.INSTANCE.toBytes(gps.getByteCount())); - put.add(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES, - PhoenixDatabaseMetaData.GUIDE_POSTS_ROW_COUNT_BYTES, timeStamp, - PLong.INSTANCE.toBytes(gps.getRowCount())); - rowColumnAdded = true; + ImmutableBytesWritable keys = gps.getGuidePosts(); + ByteArrayInputStream stream = new ByteArrayInputStream(keys.get(), keys.getOffset(), keys.getLength()); + DataInput input = new DataInputStream(stream); + PrefixByteDecoder decoder = new PrefixByteDecoder(); + try { + while (true) { + ImmutableBytesWritable ptr = decoder.decode(input); + byte[] prefix = StatisticsUtil.getRowKey(tableName, cfKey, ptr); + Put put = new Put(prefix); + if (!rowColumnAdded) { + put.add(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES, PhoenixDatabaseMetaData.GUIDE_POSTS_WIDTH_BYTES, + timeStamp, PLong.INSTANCE.toBytes(gps.getByteCount())); + put.add(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES, + PhoenixDatabaseMetaData.GUIDE_POSTS_ROW_COUNT_BYTES, timeStamp, + PLong.INSTANCE.toBytes(gps.getRowCount())); + rowColumnAdded = true; + } + // Add our empty column value so queries behave correctly + put.add(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES, QueryConstants.EMPTY_COLUMN_BYTES, timeStamp, + ByteUtil.EMPTY_BYTE_ARRAY); + mutations.add(put); } - // Add our empty column value so queries behave correctly - put.add(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES, QueryConstants.EMPTY_COLUMN_BYTES, timeStamp, - ByteUtil.EMPTY_BYTE_ARRAY); - mutations.add(put); + } catch (EOFException e) { // Ignore as this signifies we're done + + } finally { + try { + if (stream != null) { + stream.close(); + } + } catch (IOException e) {} } } } diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/ByteUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/ByteUtil.java index 64d064a75d9..44502a5478e 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/util/ByteUtil.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/util/ByteUtil.java @@ -49,6 +49,8 @@ public class ByteUtil { public static final byte[] EMPTY_BYTE_ARRAY = new byte[0]; public static final ImmutableBytesPtr EMPTY_BYTE_ARRAY_PTR = new ImmutableBytesPtr( EMPTY_BYTE_ARRAY); + public static final ImmutableBytesWritable EMPTY_IMMUTABLE_BYTE_ARRAY = new ImmutableBytesWritable( + EMPTY_BYTE_ARRAY); public static final Comparator BYTES_PTR_COMPARATOR = new Comparator() { diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/CodecUtils.java b/phoenix-core/src/main/java/org/apache/phoenix/util/CodecUtils.java new file mode 100644 index 00000000000..cc160bb53a0 --- /dev/null +++ b/phoenix-core/src/main/java/org/apache/phoenix/util/CodecUtils.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.util; + +import java.io.ByteArrayInputStream; +import java.io.DataInput; +import java.io.DataInputStream; +import java.io.DataOutput; +import java.io.DataOutputStream; +import java.io.EOFException; +import java.io.IOException; +import java.util.List; + +import org.apache.hadoop.hbase.io.ImmutableBytesWritable; + +import com.google.common.collect.Lists; + +public class CodecUtils { + + public static List decodeBytes(ImmutableBytesWritable encodedBytes, int maxLength) throws IOException { + ByteArrayInputStream stream = new ByteArrayInputStream(encodedBytes.get(), encodedBytes.getOffset(), encodedBytes.getLength()); + DataInput input = new DataInputStream(stream); + PrefixByteDecoder decoder; + if (maxLength > 0) { + decoder = new PrefixByteDecoder(maxLength); + } else { + decoder = new PrefixByteDecoder(); + } + List listOfBytes = Lists.newArrayList(); + try { + while (true) { + ImmutableBytesWritable ptr = decoder.decode(input); + // For this test, copy the bytes, but we wouldn't do this unless + // necessary for non testing + listOfBytes.add(ptr.copyBytes()); + } + } catch (EOFException e) { // Ignore as this signifies we're done + + } + return listOfBytes; + } + + public static int encodeBytes(List listOfBytes, ImmutableBytesWritable ptr) throws IOException { + TrustedByteArrayOutputStream stream = new TrustedByteArrayOutputStream(calculateSize(listOfBytes)); + DataOutput output = new DataOutputStream(stream); + PrefixByteEncoder encoder = new PrefixByteEncoder(); + for (byte[] bytes : listOfBytes) { + encoder.encode(output, bytes, 0, bytes.length); + } + stream.close(); + ptr.set(stream.getBuffer(), 0, stream.size()); + return encoder.getMaxLength(); + } + + public static int calculateSize(List listOfBytes) { + int size = 0; + for (byte[] bytes : listOfBytes) { + size += bytes.length; + } + return size; + } + + public static ImmutableBytesWritable decode(PrefixByteDecoder decoder, DataInput input) throws EOFException { + try { + ImmutableBytesWritable val= decoder.decode(input); + return val; + } catch(EOFException eof){ + throw eof; + }catch (IOException e) { + throw new RuntimeException(e); + } + } + +} diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/PrefixByteDecoder.java b/phoenix-core/src/main/java/org/apache/phoenix/util/PrefixByteDecoder.java new file mode 100644 index 00000000000..420ac83d9eb --- /dev/null +++ b/phoenix-core/src/main/java/org/apache/phoenix/util/PrefixByteDecoder.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.util; + +import java.io.DataInput; +import java.io.IOException; + +import org.apache.hadoop.hbase.io.ImmutableBytesWritable; +import org.apache.hadoop.io.WritableUtils; + +/** + * + * Prefix decoder for byte arrays. For encoding, see {@link PrefixByteEncoder}. + * + */ +public class PrefixByteDecoder { + private final int maxLength; + private final ImmutableBytesWritable previous; + + /** + * Used when the maximum length of encoded byte array is not known. Will + * cause a new byte array to be allocated for each call to {@link #decode(DataInput)}. + */ + public PrefixByteDecoder() { + maxLength = -1; + previous = new ImmutableBytesWritable(ByteUtil.EMPTY_BYTE_ARRAY); + } + + /** + * Used when the maximum length of encoded byte array is known in advance. + * Will not allocate new byte array with each call to {@link #decode(DataInput)}. + * @param maxLength maximum length needed for any call to {@link #decode(DataInput)}. + */ + public PrefixByteDecoder(int maxLength) { + this.maxLength = maxLength; + this.previous = new ImmutableBytesWritable(new byte[maxLength],0,0); + } + + /** + * Resets state of decoder if it will be used to decode bytes from a + * different DataInput. + */ + public void reset() { + previous.set(previous.get(),0,0); + } + + /** + * Decodes bytes encoded with {@link PrefixByteEncoder}. + * @param in Input from which bytes are read. + * @return Pointer containing bytes that were decoded. Note that the + * same pointer will be returned with each call, so it must be consumed + * prior to calling decode again. + * @throws IOException + */ + public ImmutableBytesWritable decode(DataInput in) throws IOException { + int prefixLen = WritableUtils.readVInt(in); + int suffixLen = WritableUtils.readVInt(in); + int length = prefixLen + suffixLen; + byte[] b; + if (maxLength == -1) { // Allocate new byte array each time + b = new byte[length]; + System.arraycopy(previous.get(), previous.getOffset(), b, 0, prefixLen); + } else { // Reuse same buffer each time + b = previous.get(); + } + in.readFully(b, prefixLen, suffixLen); + previous.set(b, 0, length); + return previous; + } +} diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/PrefixByteEncoder.java b/phoenix-core/src/main/java/org/apache/phoenix/util/PrefixByteEncoder.java new file mode 100644 index 00000000000..7a04aa0b4a6 --- /dev/null +++ b/phoenix-core/src/main/java/org/apache/phoenix/util/PrefixByteEncoder.java @@ -0,0 +1,99 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.util; + +import java.io.DataOutput; +import java.io.IOException; + +import org.apache.hadoop.hbase.io.ImmutableBytesWritable; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.io.WritableUtils; + +/** + * + * Prefix encoder for byte arrays. For decoding, see {@link PrefixByteDecoder}. + * + */ +public class PrefixByteEncoder { + private int maxLength; + private final ImmutableBytesWritable previous = new ImmutableBytesWritable(ByteUtil.EMPTY_BYTE_ARRAY); + + public PrefixByteEncoder() { + } + + /** + * Resets the state of the encoder to its initial state (i.e. forgetting + * the previous byte array that may have been encoded). + */ + public void reset() { + previous.set(ByteUtil.EMPTY_BYTE_ARRAY); + } + + /** + * @return the maximum length byte array encountered while encoding + */ + public int getMaxLength() { + return maxLength; + } + + /** + * Prefix encodes the byte array pointed to into the output stream + * @param out output stream to encode into + * @param ptr pointer to byte array to encode. + * @throws IOException + */ + public void encode(DataOutput out, ImmutableBytesWritable ptr) throws IOException { + encode(out, ptr.get(), ptr.getOffset(), ptr.getLength()); + } + + /** + * Prefix encodes the byte array into the output stream + * @param out output stream to encode into + * @param b byte array to encode + * @throws IOException + */ + public void encode(DataOutput out, byte[] b) throws IOException { + encode(out, b, 0, b.length); + } + + /** + * Prefix encodes the byte array from offset to length into output stream. + * Instead of writing the entire byte array, only the portion of the byte array + * that differs from the beginning of the previous byte array written is written. + * + * @param out output stream to encode into + * @param b byte array buffer + * @param offset offset into byte array to start encoding + * @param length length of byte array to encode + * @throws IOException + */ + public void encode(DataOutput out, byte[] b, int offset, int length) throws IOException { + int i = 0; + int prevOffset = previous.getOffset(); + byte[] prevBytes = previous.get(); + int prevLength = previous.getLength(); + int minLength = prevLength < b.length ? prevLength : b.length; + for(i = 0; (i < minLength) && (prevBytes[prevOffset + i] == b[offset + i]); i++); + WritableUtils.writeVInt(out, i); + Bytes.writeByteArray(out, b, offset + i, length - i); + previous.set(b, offset, length); + if (length > maxLength) { + maxLength = length; + } + } +} diff --git a/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanBigFilterTest.java b/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanBigFilterTest.java index a0aca4dd18c..b30f8f0505d 100644 --- a/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanBigFilterTest.java +++ b/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanBigFilterTest.java @@ -22,12 +22,11 @@ import java.sql.Connection; import java.sql.DriverManager; import java.sql.PreparedStatement; -import java.util.Collections; import java.util.Map; import java.util.Properties; import java.util.SortedMap; -import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import org.apache.hadoop.hbase.util.Bytes; import org.apache.phoenix.compile.QueryPlan; import org.apache.phoenix.end2end.Shadower; @@ -41,6 +40,7 @@ import org.apache.phoenix.schema.PTableKey; import org.apache.phoenix.schema.stats.GuidePostsInfo; import org.apache.phoenix.schema.stats.PTableStats; +import org.apache.phoenix.util.ByteUtil; import org.apache.phoenix.util.PropertiesUtil; import org.apache.phoenix.util.ReadOnlyProps; import org.junit.BeforeClass; @@ -648,10 +648,12 @@ private void testIntersect(byte[][] regionBoundaries, byte[][] guidePosts) throw stmt.execute(); final PTable table = conn.unwrap(PhoenixConnection.class).getTable(new PTableKey(null, "PERF.BIG_OLAP_DOC")); - GuidePostsInfo info = new GuidePostsInfo(0,Collections. emptyList(), 0l); + GuidePostsInfo info = new GuidePostsInfo(0, new ImmutableBytesWritable(ByteUtil.EMPTY_BYTE_ARRAY), 0l, 0, 0); for (byte[] gp : guidePosts) { - info.addGuidePost(gp, 1000); + info.encodeAndCollectGuidePost(gp, 1000); } + info.updateGuidePosts(); + info.close(); final SortedMap gpMap = Maps.newTreeMap(Bytes.BYTES_COMPARATOR); gpMap.put(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES, info); PTable tableWithStats = PTableImpl.makePTable(table, new PTableStats() { diff --git a/phoenix-core/src/test/java/org/apache/phoenix/util/PrefixByteEncoderDecoderTest.java b/phoenix-core/src/test/java/org/apache/phoenix/util/PrefixByteEncoderDecoderTest.java new file mode 100644 index 00000000000..ecb3efff94c --- /dev/null +++ b/phoenix-core/src/test/java/org/apache/phoenix/util/PrefixByteEncoderDecoderTest.java @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.util; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.io.DataOutput; +import java.io.DataOutputStream; +import java.io.IOException; +import java.util.Arrays; +import java.util.List; + +import org.apache.hadoop.hbase.io.ImmutableBytesWritable; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.io.WritableUtils; +import org.apache.phoenix.query.QueryConstants; +import org.junit.Test; + + +public class PrefixByteEncoderDecoderTest { + + static final List guideposts = Arrays.asList( + ByteUtil.concat(Bytes.toBytes("aaaaaaaaaa"), QueryConstants.SEPARATOR_BYTE_ARRAY, Bytes.toBytes(1000L), Bytes.toBytes("bbbbbbbbbb")), + ByteUtil.concat(Bytes.toBytes("aaaaaaaaaa"), QueryConstants.SEPARATOR_BYTE_ARRAY, Bytes.toBytes(1000L), Bytes.toBytes("bbbbbccccc")), + ByteUtil.concat(Bytes.toBytes("aaaaaaaaaa"), QueryConstants.SEPARATOR_BYTE_ARRAY, Bytes.toBytes(2000L), Bytes.toBytes("bbbbbbbbbb")), + ByteUtil.concat(Bytes.toBytes("bbbbbbbbbb"), QueryConstants.SEPARATOR_BYTE_ARRAY, Bytes.toBytes(1000L), Bytes.toBytes("bbbbbbbbbb")), + ByteUtil.concat(Bytes.toBytes("bbbbbbbbbb"), QueryConstants.SEPARATOR_BYTE_ARRAY, Bytes.toBytes(2000L), Bytes.toBytes("bbbbbbbbbb")), + ByteUtil.concat(Bytes.toBytes("bbbbbbbbbb"), QueryConstants.SEPARATOR_BYTE_ARRAY, Bytes.toBytes(2000L), Bytes.toBytes("c")), + ByteUtil.concat(Bytes.toBytes("bbbbbbbbbbb"), QueryConstants.SEPARATOR_BYTE_ARRAY, Bytes.toBytes(1000L), Bytes.toBytes("bbbbbbbbbb")), + ByteUtil.concat(Bytes.toBytes("d"), QueryConstants.SEPARATOR_BYTE_ARRAY, Bytes.toBytes(1000L), Bytes.toBytes("bbbbbbbbbb")), + ByteUtil.concat(Bytes.toBytes("d"), QueryConstants.SEPARATOR_BYTE_ARRAY, Bytes.toBytes(1000L), Bytes.toBytes("bbbbbbbbbbc")), + ByteUtil.concat(Bytes.toBytes("e"), QueryConstants.SEPARATOR_BYTE_ARRAY, Bytes.toBytes(1000L), Bytes.toBytes("bbbbbbbbbb")) + ); + + @Test + public void testEncode() throws IOException { + List listOfBytes = Arrays.asList(Bytes.toBytes("aaaaa"), Bytes.toBytes("aaaabb")); + ImmutableBytesWritable ptr = new ImmutableBytesWritable(); + int maxLength = CodecUtils.encodeBytes(listOfBytes, ptr); + assertEquals(6, maxLength); + TrustedByteArrayOutputStream stream = new TrustedByteArrayOutputStream(CodecUtils.calculateSize(listOfBytes)); + DataOutput output = new DataOutputStream(stream); + WritableUtils.writeVInt(output, 0); + WritableUtils.writeVInt(output, 5); + output.write(Bytes.toBytes("aaaaa")); // No space savings on first key + WritableUtils.writeVInt(output, 4); + WritableUtils.writeVInt(output, 2); + output.write(Bytes.toBytes("bb")); // Only writes part of second key that's different + assertArrayEquals(stream.toByteArray(), ptr.copyBytes()); + } + + @Test + public void testEncodeDecodeWithSingleBuffer() throws IOException { + testEncodeDecode(true); + } + + @Test + public void testEncodeDecodeWithNewBuffer() throws IOException { + testEncodeDecode(false); + } + + private void testEncodeDecode(boolean useSingleBuffer) throws IOException { + ImmutableBytesWritable ptr = new ImmutableBytesWritable(); + int maxLength = CodecUtils.encodeBytes(guideposts, ptr); + int encodedSize = ptr.getLength(); + int unencodedSize = CodecUtils.calculateSize(guideposts); + assertTrue(encodedSize < unencodedSize); + List listOfBytes = CodecUtils.decodeBytes(ptr, useSingleBuffer ? maxLength : -1); + assertListByteArraysEquals(guideposts, listOfBytes); + } + + private static void assertListByteArraysEquals(List listOfBytes1, List listOfBytes2) { + assertEquals(listOfBytes1.size(), listOfBytes2.size()); + for (int i = 0; i < listOfBytes1.size(); i++) { + assertArrayEquals(listOfBytes1.get(i), listOfBytes2.get(i)); + } + } + +} diff --git a/phoenix-protocol/src/main/PGuidePosts.proto b/phoenix-protocol/src/main/PGuidePosts.proto index 047a658de3e..d0c7c9bdfd7 100644 --- a/phoenix-protocol/src/main/PGuidePosts.proto +++ b/phoenix-protocol/src/main/PGuidePosts.proto @@ -24,7 +24,9 @@ option java_generic_services = true; option java_generate_equals_and_hash = true; option optimize_for = SPEED; message PGuidePosts { - repeated bytes guidePosts = 1; + optional bytes guidePosts = 1; optional int64 byteCount = 2; optional int64 rowCount = 3; + optional int32 maxLength = 4; + optional int32 guidePostsCount = 5; } \ No newline at end of file diff --git a/phoenix-protocol/src/main/PTable.proto b/phoenix-protocol/src/main/PTable.proto index 3048a40cd99..228c03c871b 100644 --- a/phoenix-protocol/src/main/PTable.proto +++ b/phoenix-protocol/src/main/PTable.proto @@ -52,11 +52,12 @@ message PColumn { message PTableStats { required bytes key = 1; - repeated bytes values = 2; + optional bytes guidePosts = 2; optional int64 guidePostsByteCount = 3; optional int64 keyBytesCount = 4; optional int32 guidePostsCount = 5; optional PGuidePosts pGuidePosts = 6; + optional int32 maxLength = 7; } message PTable { From 3430d5e8fdc59cc187a454209b8af9a6c21d0a0e Mon Sep 17 00:00:00 2001 From: Ankit Singhal Date: Sat, 16 Jan 2016 02:22:29 +0530 Subject: [PATCH 02/47] PHOENIX-2417 Review comments --- .../apache/phoenix/compile/ScanRanges.java | 10 +++--- .../phoenix/iterate/BaseResultIterators.java | 36 ++++++++----------- .../org/apache/phoenix/schema/PTableImpl.java | 3 +- .../phoenix/schema/stats/GuidePostsInfo.java | 11 ++---- .../phoenix/schema/stats/PTableStatsImpl.java | 18 ++-------- .../schema/stats/StatisticsWriter.java | 9 ++--- .../org/apache/phoenix/util/CodecUtils.java | 31 +++++++++++----- .../phoenix/util/PrefixByteDecoder.java | 9 +++-- .../compile/ScanRangesIntersectTest.java | 4 ++- 9 files changed, 62 insertions(+), 69 deletions(-) diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/ScanRanges.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/ScanRanges.java index 91d07ac898d..4d343f342e8 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/compile/ScanRanges.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/ScanRanges.java @@ -237,9 +237,7 @@ private static byte[] stripPrefix(byte[] key, int keyOffset) { return temp; } - public Scan intersectScan(Scan scan, final ImmutableBytesWritable originalStartKeyPtr, final ImmutableBytesWritable originalStopKeyPtr, final int keyOffset, boolean crossesRegionBoundary) { - byte[] originalStartKey= originalStartKeyPtr.get(); - byte[] originalStopKey= originalStopKeyPtr.get(); + public Scan intersectScan(Scan scan, final byte[] originalStartKey, final byte[] originalStopKey, final int keyOffset, boolean crossesRegionBoundary) { byte[] startKey = originalStartKey; byte[] stopKey = originalStopKey; if (stopKey.length > 0 && Bytes.compareTo(startKey, stopKey) >= 0) { @@ -386,8 +384,8 @@ public Scan intersectScan(Scan scan, final ImmutableBytesWritable originalStartK if (scanStopKey.length > 0 && Bytes.compareTo(scanStartKey, scanStopKey) >= 0) { return null; } - newScan.setStartRow(Bytes.copy(scanStartKey)); - newScan.setStopRow(Bytes.copy(scanStopKey)); + newScan.setStartRow(scanStartKey); + newScan.setStopRow(scanStopKey); if(keyOffset > 0) { newScan.setAttribute(STARTKEY_OFFSET, Bytes.toBytes(keyOffset)); } @@ -418,7 +416,7 @@ public boolean intersects(byte[] lowerInclusiveKey, byte[] upperExclusiveKey, in } //return filter.hasIntersect(lowerInclusiveKey, upperExclusiveKey); - return intersectScan(null, new ImmutableBytesWritable(lowerInclusiveKey), new ImmutableBytesWritable(upperExclusiveKey), keyOffset, crossesRegionBoundary) == HAS_INTERSECTION; + return intersectScan(null, lowerInclusiveKey, upperExclusiveKey, keyOffset, crossesRegionBoundary) == HAS_INTERSECTION; } public SkipScanFilter getSkipScanFilter() { diff --git a/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java b/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java index 1a5cdc55cfb..e17ce86bf08 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java @@ -26,7 +26,6 @@ import java.io.DataInput; import java.io.DataInputStream; import java.io.EOFException; -import java.io.IOException; import java.sql.SQLException; import java.util.ArrayList; import java.util.Collections; @@ -508,21 +507,18 @@ private List> getParallelScans(byte[] startKey, byte[] stopKey) throw if (gpsSize > 0) { stream = new ByteArrayInputStream(guidePosts.get(), guidePosts.getOffset(), guidePosts.getLength()); input = new DataInputStream(stream); - if (gps.getMaxLength() > 0) { - decoder = new PrefixByteDecoder(gps.getMaxLength()); - } else { - decoder = new PrefixByteDecoder(); - } + decoder = new PrefixByteDecoder(gps.getMaxLength()); try { - currentGuidePost = CodecUtils.decode(decoder, input); - while (currentKey.getLength() != 0 && Bytes.compareTo(currentKey.get(), currentGuidePost.get()) <= 0) { - currentGuidePost = CodecUtils.decode(decoder, input); + while (currentKey.compareTo(currentGuidePost = CodecUtils.decode(decoder, input)) >= 0) { guideIndex++; } } catch (EOFException e) {} } + byte[] currentKeyBytes = ByteUtil.copyKeyBytesIfNecessary(currentKey); + // Merge bisect with guideposts for all but the last region while (regionIndex <= stopIndex) { + byte[] currentGuidePostBytes = currentGuidePost.copyBytes(); byte[] endKey, endRegionKey = EMPTY_BYTE_ARRAY; if (regionIndex == stopIndex) { endKey = stopKey; @@ -536,17 +532,17 @@ private List> getParallelScans(byte[] startKey, byte[] stopKey) throw keyOffset = ScanUtil.getRowKeyOffset(regionInfo.getStartKey(), endRegionKey); } try { - while (guideIndex < gpsSize - && (Bytes.compareTo(currentGuidePost.get(), - endKey) <= 0 || endKey.length == 0)) { - Scan newScan = scanRanges.intersectScan(scan, currentKey, currentGuidePost, keyOffset, false); - scans = addNewScan(parallelScans, scans, newScan, currentGuidePost.get(), false, regionLocation); - currentKey = new ImmutableBytesWritable(currentGuidePost.copyBytes()); + while (guideIndex < gpsSize && (currentGuidePost.compareTo(endKey) <= 0 || endKey.length == 0)) { + Scan newScan = scanRanges.intersectScan(scan, currentKeyBytes, currentGuidePostBytes, keyOffset, + false); + scans = addNewScan(parallelScans, scans, newScan, currentGuidePostBytes, false, regionLocation); + currentKeyBytes = currentGuidePost.copyBytes(); currentGuidePost = CodecUtils.decode(decoder, input); + currentGuidePostBytes = ByteUtil.copyKeyBytesIfNecessary(currentGuidePost); guideIndex++; } } catch (EOFException e) {} - Scan newScan = scanRanges.intersectScan(scan, currentKey, new ImmutableBytesWritable(endKey,0,endKey.length), keyOffset, true); + Scan newScan = scanRanges.intersectScan(scan, currentKeyBytes, endKey, keyOffset, true); if (isLocalIndex) { if (newScan != null) { newScan.setAttribute(EXPECTED_UPPER_REGION_KEY, endRegionKey); @@ -555,17 +551,13 @@ private List> getParallelScans(byte[] startKey, byte[] stopKey) throw } } scans = addNewScan(parallelScans, scans, newScan, endKey, true, regionLocation); - currentKey = new ImmutableBytesWritable(endKey, 0, endKey.length); + currentKeyBytes = endKey; regionIndex++; } if (!scans.isEmpty()) { // Add any remaining scans parallelScans.add(scans); } - if (stream != null) { - try { - stream.close(); - } catch (IOException e) {} - } + CodecUtils.close(stream); return parallelScans; } diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java index e1e76e5179a..90ddd786dba 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java @@ -75,6 +75,7 @@ import com.google.common.collect.ListMultimap; import com.google.common.collect.Lists; import com.google.common.collect.Maps; +import com.google.protobuf.HBaseZeroCopyByteString; import com.sun.istack.NotNull; import co.cask.tephra.TxConstants; @@ -1009,7 +1010,7 @@ public static PTable createFromProto(PTableProtos.PTable table) { int maxLength = pGuidePosts.getMaxLength(); int guidePostsCount = pGuidePosts.getGuidePostsCount(); GuidePostsInfo info = new GuidePostsInfo(guidePostsByteCount, - new ImmutableBytesWritable(pGuidePosts.getGuidePosts().toByteArray()), rowCount, maxLength, guidePostsCount); + new ImmutableBytesWritable(HBaseZeroCopyByteString.zeroCopyGetBytes(pGuidePosts.getGuidePosts())), rowCount, maxLength, guidePostsCount); tableGuidePosts.put(pTableStatsProto.getKey().toByteArray(), info); } PTableStats stats = new PTableStatsImpl(tableGuidePosts, table.getStatsTimeStamp()); diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsInfo.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsInfo.java index 00fb3499385..b7f5c49f87c 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsInfo.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsInfo.java @@ -23,6 +23,7 @@ import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import org.apache.hadoop.hbase.util.Bytes; import org.apache.phoenix.util.ByteUtil; +import org.apache.phoenix.util.CodecUtils; import org.apache.phoenix.util.PrefixByteEncoder; import org.apache.phoenix.util.TrustedByteArrayOutputStream; /** @@ -71,7 +72,7 @@ public int getMaxLength() { * @param rowCount */ public GuidePostsInfo(long byteCount, ImmutableBytesWritable guidePosts, long rowCount, int maxLength, int guidePostsCount) { - this.guidePosts = guidePosts; + this.guidePosts = new ImmutableBytesWritable(guidePosts); this.maxLength = maxLength; this.byteCount = byteCount; this.rowCount = rowCount; @@ -141,13 +142,7 @@ public boolean encodeAndCollectGuidePost(byte[] row, long byteCount){ } public void close() { - if (stream != null) { - try { - stream.close(); - } catch (IOException e) { - //exception can be ignored - } - } + CodecUtils.close(stream); } /* diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/PTableStatsImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/PTableStatsImpl.java index c6b8d921918..81c73feabd0 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/PTableStatsImpl.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/PTableStatsImpl.java @@ -21,7 +21,6 @@ import java.io.DataInput; import java.io.DataInputStream; import java.io.EOFException; -import java.io.IOException; import java.util.Map; import java.util.SortedMap; import java.util.TreeMap; @@ -80,12 +79,7 @@ public String toString() { try { if (keys.getLength() != 0) { DataInput input = new DataInputStream(stream); - PrefixByteDecoder decoder; - if (entry.getValue().getMaxLength() > 0) { - decoder = new PrefixByteDecoder(entry.getValue().getMaxLength()); - } else { - decoder = new PrefixByteDecoder(); - } + PrefixByteDecoder decoder = new PrefixByteDecoder(entry.getValue().getMaxLength()); try { while (true) { ImmutableBytesWritable ptr = CodecUtils.decode(decoder, input); @@ -95,19 +89,13 @@ public String toString() { } catch (EOFException e) { // Ignore as this signifies we're done } finally { - try { - stream.close(); - } catch (IOException e) {} + CodecUtils.close(stream); } buf.setLength(buf.length() - 1); } buf.append(")"); } finally { - if (stream != null) { - try { - stream.close(); - } catch (IOException e) {} - } + CodecUtils.close(stream); } } buf.append("]"); diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsWriter.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsWriter.java index 1dcfb3df265..188434ac356 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsWriter.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsWriter.java @@ -49,6 +49,7 @@ import org.apache.phoenix.schema.types.PDate; import org.apache.phoenix.schema.types.PLong; import org.apache.phoenix.util.ByteUtil; +import org.apache.phoenix.util.CodecUtils; import org.apache.phoenix.util.PrefixByteDecoder; import org.apache.phoenix.util.ServerUtil; import org.apache.phoenix.util.TimeKeeper; @@ -143,7 +144,7 @@ public void addStats(StatisticsCollector tracker, ImmutableBytesPtr cfKey, List< ImmutableBytesWritable keys = gps.getGuidePosts(); ByteArrayInputStream stream = new ByteArrayInputStream(keys.get(), keys.getOffset(), keys.getLength()); DataInput input = new DataInputStream(stream); - PrefixByteDecoder decoder = new PrefixByteDecoder(); + PrefixByteDecoder decoder = new PrefixByteDecoder(gps.getMaxLength()); try { while (true) { ImmutableBytesWritable ptr = decoder.decode(input); @@ -165,11 +166,7 @@ public void addStats(StatisticsCollector tracker, ImmutableBytesPtr cfKey, List< } catch (EOFException e) { // Ignore as this signifies we're done } finally { - try { - if (stream != null) { - stream.close(); - } - } catch (IOException e) {} + CodecUtils.close(stream); } } } diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/CodecUtils.java b/phoenix-core/src/main/java/org/apache/phoenix/util/CodecUtils.java index cc160bb53a0..f4b330269e9 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/util/CodecUtils.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/util/CodecUtils.java @@ -18,6 +18,7 @@ package org.apache.phoenix.util; import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; import java.io.DataInput; import java.io.DataInputStream; import java.io.DataOutput; @@ -35,12 +36,7 @@ public class CodecUtils { public static List decodeBytes(ImmutableBytesWritable encodedBytes, int maxLength) throws IOException { ByteArrayInputStream stream = new ByteArrayInputStream(encodedBytes.get(), encodedBytes.getOffset(), encodedBytes.getLength()); DataInput input = new DataInputStream(stream); - PrefixByteDecoder decoder; - if (maxLength > 0) { - decoder = new PrefixByteDecoder(maxLength); - } else { - decoder = new PrefixByteDecoder(); - } + PrefixByteDecoder decoder = new PrefixByteDecoder(maxLength); List listOfBytes = Lists.newArrayList(); try { while (true) { @@ -62,7 +58,7 @@ public static int encodeBytes(List listOfBytes, ImmutableBytesWritable p for (byte[] bytes : listOfBytes) { encoder.encode(output, bytes, 0, bytes.length); } - stream.close(); + close(stream); ptr.set(stream.getBuffer(), 0, stream.size()); return encoder.getMaxLength(); } @@ -85,5 +81,24 @@ public static ImmutableBytesWritable decode(PrefixByteDecoder decoder, DataInput throw new RuntimeException(e); } } - + + public static void close(ByteArrayInputStream stream) { + if (stream != null) { + try { + stream.close(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + } + + public static void close(ByteArrayOutputStream stream) { + if (stream != null) { + try { + stream.close(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + } } diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/PrefixByteDecoder.java b/phoenix-core/src/main/java/org/apache/phoenix/util/PrefixByteDecoder.java index 420ac83d9eb..565f0bcaa7a 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/util/PrefixByteDecoder.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/util/PrefixByteDecoder.java @@ -47,8 +47,13 @@ public PrefixByteDecoder() { * @param maxLength maximum length needed for any call to {@link #decode(DataInput)}. */ public PrefixByteDecoder(int maxLength) { - this.maxLength = maxLength; - this.previous = new ImmutableBytesWritable(new byte[maxLength],0,0); + if (maxLength > 0) { + this.maxLength = maxLength; + this.previous = new ImmutableBytesWritable(new byte[maxLength], 0, 0); + } else { + this.maxLength = -1; + previous = new ImmutableBytesWritable(ByteUtil.EMPTY_BYTE_ARRAY); + } } /** diff --git a/phoenix-core/src/test/java/org/apache/phoenix/compile/ScanRangesIntersectTest.java b/phoenix-core/src/test/java/org/apache/phoenix/compile/ScanRangesIntersectTest.java index c2bbc06694f..a3e3e2411d5 100644 --- a/phoenix-core/src/test/java/org/apache/phoenix/compile/ScanRangesIntersectTest.java +++ b/phoenix-core/src/test/java/org/apache/phoenix/compile/ScanRangesIntersectTest.java @@ -27,6 +27,7 @@ import java.util.List; import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import org.apache.phoenix.filter.SkipScanFilter; import org.apache.phoenix.query.KeyRange; import org.apache.phoenix.schema.types.PVarchar; @@ -51,7 +52,8 @@ private static void assertIntersect(ScanRanges ranges, String lowerRange, String scan.setFilter(ranges.getSkipScanFilter()); byte[] startKey = lowerRange == null ? KeyRange.UNBOUND : PVarchar.INSTANCE.toBytes(lowerRange); byte[] stopKey = upperRange == null ? KeyRange.UNBOUND : PVarchar.INSTANCE.toBytes(upperRange); - Scan newScan = ranges.intersectScan(scan, startKey, stopKey, 0, true); + Scan newScan = ranges.intersectScan(scan, new ImmutableBytesWritable(startKey, 0, startKey.length), + new ImmutableBytesWritable(stopKey, 0, stopKey.length), 0, true); if (expectedPoints.length == 0) { assertNull(newScan); } else { From ffafe106b36bec71f54e16de8323d3cb682e1157 Mon Sep 17 00:00:00 2001 From: Ankit Singhal Date: Sat, 16 Jan 2016 02:25:20 +0530 Subject: [PATCH 03/47] PHOENIX-2417 Minor nit --- .../org/apache/phoenix/compile/ScanRangesIntersectTest.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/phoenix-core/src/test/java/org/apache/phoenix/compile/ScanRangesIntersectTest.java b/phoenix-core/src/test/java/org/apache/phoenix/compile/ScanRangesIntersectTest.java index a3e3e2411d5..c2bbc06694f 100644 --- a/phoenix-core/src/test/java/org/apache/phoenix/compile/ScanRangesIntersectTest.java +++ b/phoenix-core/src/test/java/org/apache/phoenix/compile/ScanRangesIntersectTest.java @@ -27,7 +27,6 @@ import java.util.List; import org.apache.hadoop.hbase.client.Scan; -import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import org.apache.phoenix.filter.SkipScanFilter; import org.apache.phoenix.query.KeyRange; import org.apache.phoenix.schema.types.PVarchar; @@ -52,8 +51,7 @@ private static void assertIntersect(ScanRanges ranges, String lowerRange, String scan.setFilter(ranges.getSkipScanFilter()); byte[] startKey = lowerRange == null ? KeyRange.UNBOUND : PVarchar.INSTANCE.toBytes(lowerRange); byte[] stopKey = upperRange == null ? KeyRange.UNBOUND : PVarchar.INSTANCE.toBytes(upperRange); - Scan newScan = ranges.intersectScan(scan, new ImmutableBytesWritable(startKey, 0, startKey.length), - new ImmutableBytesWritable(stopKey, 0, stopKey.length), 0, true); + Scan newScan = ranges.intersectScan(scan, startKey, stopKey, 0, true); if (expectedPoints.length == 0) { assertNull(newScan); } else { From 436c6e6135b03d412c05460c76c6c2368d129005 Mon Sep 17 00:00:00 2001 From: Ankit Singhal Date: Sat, 16 Jan 2016 03:17:20 +0530 Subject: [PATCH 04/47] PHOENIX-2417 --- .../org/apache/phoenix/iterate/BaseResultIterators.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java b/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java index e17ce86bf08..8977217bbc1 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java @@ -509,12 +509,13 @@ private List> getParallelScans(byte[] startKey, byte[] stopKey) throw input = new DataInputStream(stream); decoder = new PrefixByteDecoder(gps.getMaxLength()); try { - while (currentKey.compareTo(currentGuidePost = CodecUtils.decode(decoder, input)) >= 0) { + while (currentKey.compareTo(currentGuidePost = CodecUtils.decode(decoder, input)) >= 0 + && currentKey.getLength() != 0) { guideIndex++; } } catch (EOFException e) {} } - byte[] currentKeyBytes = ByteUtil.copyKeyBytesIfNecessary(currentKey); + byte[] currentKeyBytes = currentKey.copyBytes(); // Merge bisect with guideposts for all but the last region while (regionIndex <= stopIndex) { @@ -538,7 +539,7 @@ private List> getParallelScans(byte[] startKey, byte[] stopKey) throw scans = addNewScan(parallelScans, scans, newScan, currentGuidePostBytes, false, regionLocation); currentKeyBytes = currentGuidePost.copyBytes(); currentGuidePost = CodecUtils.decode(decoder, input); - currentGuidePostBytes = ByteUtil.copyKeyBytesIfNecessary(currentGuidePost); + currentGuidePostBytes = currentGuidePost.copyBytes(); guideIndex++; } } catch (EOFException e) {} From 3c2ec562f6c347e5bfc0c1c9bc9981f8d84644ff Mon Sep 17 00:00:00 2001 From: maryannxue Date: Fri, 15 Jan 2016 22:27:42 -0500 Subject: [PATCH 05/47] PHOENIX-2601 Query result is incorrect when both index hint and limit are used --- .../index/GlobalIndexOptimizationIT.java | 23 +++++++++++++++++++ .../phoenix/optimize/QueryOptimizer.java | 2 +- 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/GlobalIndexOptimizationIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/GlobalIndexOptimizationIT.java index b97176f0719..10634b8b86e 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/GlobalIndexOptimizationIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/GlobalIndexOptimizationIT.java @@ -157,6 +157,29 @@ private void testOptimization(Integer saltBuckets) throws Exception { assertEquals("a", rs.getString("v1")); assertFalse(rs.next()); + query = "SELECT /*+ INDEX(" + TestUtil.DEFAULT_DATA_TABLE_NAME + " " + TestUtil.DEFAULT_INDEX_TABLE_NAME + ")*/ * FROM " + TestUtil.DEFAULT_DATA_TABLE_NAME +" where v1='a' limit 1"; + rs = conn1.createStatement().executeQuery("EXPLAIN "+ query); + + expected = + "CLIENT PARALLEL 1-WAY FULL SCAN OVER " + TestUtil.DEFAULT_DATA_TABLE_NAME + "\n" + + "CLIENT 1 ROW LIMIT\n" + + " SKIP-SCAN-JOIN TABLE 0\n" + + " CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + TestUtil.DEFAULT_INDEX_TABLE_NAME + " \\['a'\\]\n" + + " SERVER FILTER BY FIRST KEY ONLY\n" + + " DYNAMIC SERVER FILTER BY \\(\"T.T_ID\", \"T.K1\", \"T.K2\"\\) IN \\(\\(\\$\\d+.\\$\\d+, \\$\\d+.\\$\\d+, \\$\\d+.\\$\\d+\\)\\)\n" + + " JOIN-SCANNER 1 ROW LIMIT"; + actual = QueryUtil.getExplainPlan(rs); + assertTrue("Expected:\n" + expected + "\nbut got\n" + actual, Pattern.matches(expected, actual)); + + rs = conn1.createStatement().executeQuery(query); + assertTrue(rs.next()); + assertEquals("f", rs.getString("t_id")); + assertEquals(1, rs.getInt("k1")); + assertEquals(2, rs.getInt("k2")); + assertEquals(3, rs.getInt("k3")); + assertEquals("a", rs.getString("v1")); + assertFalse(rs.next()); + query = "SELECT /*+ INDEX(" + TestUtil.DEFAULT_DATA_TABLE_NAME + " " + TestUtil.DEFAULT_INDEX_TABLE_NAME + ")*/ t_id, k1, k2, k3, V1 from " + TestUtil.DEFAULT_DATA_TABLE_FULL_NAME + " where v1<='z' and k3 > 1 order by V1,t_id"; rs = conn1.createStatement().executeQuery("EXPLAIN " + query); diff --git a/phoenix-core/src/main/java/org/apache/phoenix/optimize/QueryOptimizer.java b/phoenix-core/src/main/java/org/apache/phoenix/optimize/QueryOptimizer.java index c5ff3c6abf7..adc3c2d1464 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/optimize/QueryOptimizer.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/optimize/QueryOptimizer.java @@ -285,7 +285,7 @@ private static QueryPlan addPlan(PhoenixStatement statement, SelectStatement sel aliasedNodes.add(FACTORY.aliasedNode(null, indexColNode)); nodes.add(new ColumnParseNode(null, '"' + column.getName().getString() + '"')); } - SelectStatement innerSelect = FACTORY.select(indexSelect.getFrom(), indexSelect.getHint(), false, aliasedNodes, where, null, null, null, indexSelect.getLimit(), indexSelect.getBindCount(), false, indexSelect.hasSequence(), Collections.emptyList(), indexSelect.getUdfParseNodes()); + SelectStatement innerSelect = FACTORY.select(indexSelect.getFrom(), indexSelect.getHint(), false, aliasedNodes, where, null, null, null, null, indexSelect.getBindCount(), false, indexSelect.hasSequence(), Collections.emptyList(), indexSelect.getUdfParseNodes()); ParseNode outerWhere = FACTORY.in(nodes.size() == 1 ? nodes.get(0) : FACTORY.rowValueConstructor(nodes), FACTORY.subquery(innerSelect, false), false, true); ParseNode extractedCondition = whereRewriter.getExtractedCondition(); if (extractedCondition != null) { From c2ee4f7fd7dc80b511ba23e0431a3b8ffeae26a3 Mon Sep 17 00:00:00 2001 From: Samarth Date: Fri, 15 Jan 2016 20:17:08 -0800 Subject: [PATCH 06/47] PHOENIX-1261 Addendum to fix test failures --- .../phoenix/coprocessor/BaseScannerRegionObserver.java | 2 +- .../coprocessor/UngroupedAggregateRegionObserver.java | 2 +- .../main/java/org/apache/phoenix/schema/MetaDataClient.java | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/BaseScannerRegionObserver.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/BaseScannerRegionObserver.java index 236c2dca717..034848b1cac 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/BaseScannerRegionObserver.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/BaseScannerRegionObserver.java @@ -93,7 +93,7 @@ abstract public class BaseScannerRegionObserver extends BaseRegionObserver { public static final String GUIDEPOST_PER_REGION = "_GUIDEPOST_PER_REGION"; public static final String UPGRADE_DESC_ROW_KEY = "_UPGRADE_DESC_ROW_KEY"; public static final String SCAN_REGION_SERVER = "_SCAN_REGION_SERVER"; - public static final String RUN_UPDATE_STATS_ASYNC = "_RunUpdateStatsAsync"; + public static final String RUN_UPDATE_STATS_ASYNC_ATTRIB = "_RunUpdateStatsAsync"; /** * Attribute name used to pass custom annotations in Scans and Mutations (later). Custom annotations diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java index 1d67994747e..f332e60b960 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java @@ -666,7 +666,7 @@ private RegionScanner collectStats(final RegionScanner innerScanner, StatisticsC final Region region, final Scan scan, Configuration config) throws IOException { StatsCollectionCallable callable = new StatsCollectionCallable(stats, region, innerScanner, config); - byte[] asyncBytes = scan.getAttribute(BaseScannerRegionObserver.RUN_UPDATE_STATS_ASYNC); + byte[] asyncBytes = scan.getAttribute(BaseScannerRegionObserver.RUN_UPDATE_STATS_ASYNC_ATTRIB); boolean async = false; if (asyncBytes != null) { async = Bytes.toBoolean(asyncBytes); diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java index 995be86db77..f1bb582fa72 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java @@ -22,7 +22,7 @@ import static com.google.common.collect.Sets.newLinkedHashSetWithExpectedSize; import static org.apache.hadoop.hbase.HColumnDescriptor.TTL; import static org.apache.phoenix.coprocessor.BaseScannerRegionObserver.ANALYZE_TABLE; -import static org.apache.phoenix.coprocessor.BaseScannerRegionObserver.RUN_UPDATE_STATS_ASYNC; +import static org.apache.phoenix.coprocessor.BaseScannerRegionObserver.RUN_UPDATE_STATS_ASYNC_ATTRIB; import static org.apache.phoenix.exception.SQLExceptionCode.INSUFFICIENT_MULTI_TENANT_COLUMNS; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.ARG_POSITION; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.ARRAY_SIZE; @@ -955,8 +955,8 @@ public boolean isTransactional() { Scan scan = plan.getContext().getScan(); scan.setCacheBlocks(false); scan.setAttribute(ANALYZE_TABLE, TRUE_BYTES); - boolean runUpdateStatsAsync = props.getBoolean(RUN_UPDATE_STATS_ASYNC, DEFAULT_RUN_UPDATE_STATS_ASYNC); - scan.setAttribute(RUN_UPDATE_STATS_ASYNC, runUpdateStatsAsync ? TRUE_BYTES : FALSE_BYTES); + boolean runUpdateStatsAsync = props.getBoolean(QueryServices.RUN_UPDATE_STATS_ASYNC, DEFAULT_RUN_UPDATE_STATS_ASYNC); + scan.setAttribute(RUN_UPDATE_STATS_ASYNC_ATTRIB, runUpdateStatsAsync ? TRUE_BYTES : FALSE_BYTES); if (statsProps != null) { Object gp_width = statsProps.get(QueryServices.STATS_GUIDEPOST_WIDTH_BYTES_ATTRIB); if (gp_width != null) { From c584029b9face0acb0513d328d68ec42b25fc011 Mon Sep 17 00:00:00 2001 From: James Taylor Date: Fri, 15 Jan 2016 22:09:39 -0800 Subject: [PATCH 07/47] PHOENIX-2600 NPE on immutable index creation over transactional table --- .../org/apache/phoenix/end2end/UpsertSelectIT.java | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpsertSelectIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpsertSelectIT.java index 364b4232c88..30de4de4816 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpsertSelectIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpsertSelectIT.java @@ -1376,7 +1376,9 @@ public void testUpsertSelectWithFixedWidthNullByteSizeArray() throws Exception { @Test public void testParallelUpsertSelect() throws Exception { + long ts = nextTimestamp(); Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); + props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts)); props.setProperty(QueryServices.MUTATE_BATCH_SIZE_ATTRIB, Integer.toString(3)); props.setProperty(QueryServices.SCAN_CACHE_SIZE_ATTRIB, Integer.toString(3)); props.setProperty(QueryServices.SCAN_RESULT_CHUNK_SIZE, Integer.toString(3)); @@ -1385,11 +1387,17 @@ public void testParallelUpsertSelect() throws Exception { conn.createStatement().execute("CREATE SEQUENCE S1"); conn.createStatement().execute("CREATE TABLE SALTEDT1 (pk INTEGER PRIMARY KEY, val INTEGER) SALT_BUCKETS=4"); conn.createStatement().execute("CREATE TABLE T2 (pk INTEGER PRIMARY KEY, val INTEGER)"); - + conn.close(); + + props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 10)); + conn = DriverManager.getConnection(getUrl(), props); for (int i = 0; i < 100; i++) { conn.createStatement().execute("UPSERT INTO SALTEDT1 VALUES (NEXT VALUE FOR S1, " + (i%10) + ")"); } conn.commit(); + conn.close(); + props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 20)); + conn = DriverManager.getConnection(getUrl(), props); conn.setAutoCommit(true); int upsertCount = conn.createStatement().executeUpdate("UPSERT INTO T2 SELECT pk, val FROM SALTEDT1"); assertEquals(100,upsertCount); From 6ad4a375b2a57c3e3d5e628facb1c43214c17023 Mon Sep 17 00:00:00 2001 From: Ankit Singhal Date: Sat, 16 Jan 2016 21:35:07 +0530 Subject: [PATCH 08/47] review comments second round --- .../coprocessor/generated/PTableProtos.java | 383 ++++++++++++------ .../phoenix/iterate/BaseResultIterators.java | 5 +- .../phoenix/schema/stats/GuidePostsInfo.java | 76 +--- .../schema/stats/GuidePostsInfoWriter.java | 122 ++++++ .../schema/stats/StatisticsCollector.java | 50 +-- .../phoenix/schema/stats/StatisticsUtil.java | 41 +- phoenix-protocol/src/main/PTable.proto | 5 +- 7 files changed, 447 insertions(+), 235 deletions(-) create mode 100644 phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsInfoWriter.java diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/PTableProtos.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/PTableProtos.java index 1c5c0388f1b..04f8849cfda 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/PTableProtos.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/PTableProtos.java @@ -1843,15 +1843,19 @@ public interface PTableStatsOrBuilder */ com.google.protobuf.ByteString getKey(); - // optional bytes guidePosts = 2; + // repeated bytes values = 2; /** - * optional bytes guidePosts = 2; + * repeated bytes values = 2; */ - boolean hasGuidePosts(); + java.util.List getValuesList(); /** - * optional bytes guidePosts = 2; + * repeated bytes values = 2; */ - com.google.protobuf.ByteString getGuidePosts(); + int getValuesCount(); + /** + * repeated bytes values = 2; + */ + com.google.protobuf.ByteString getValues(int index); // optional int64 guidePostsByteCount = 3; /** @@ -1897,13 +1901,23 @@ public interface PTableStatsOrBuilder */ org.apache.phoenix.coprocessor.generated.PGuidePostsProtos.PGuidePostsOrBuilder getPGuidePostsOrBuilder(); - // optional int32 maxLength = 7; + // optional bytes guidePosts = 7; + /** + * optional bytes guidePosts = 7; + */ + boolean hasGuidePosts(); /** - * optional int32 maxLength = 7; + * optional bytes guidePosts = 7; + */ + com.google.protobuf.ByteString getGuidePosts(); + + // optional int32 maxLength = 8; + /** + * optional int32 maxLength = 8; */ boolean hasMaxLength(); /** - * optional int32 maxLength = 7; + * optional int32 maxLength = 8; */ int getMaxLength(); } @@ -1964,28 +1978,31 @@ private PTableStats( break; } case 18: { - bitField0_ |= 0x00000002; - guidePosts_ = input.readBytes(); + if (!((mutable_bitField0_ & 0x00000002) == 0x00000002)) { + values_ = new java.util.ArrayList(); + mutable_bitField0_ |= 0x00000002; + } + values_.add(input.readBytes()); break; } case 24: { - bitField0_ |= 0x00000004; + bitField0_ |= 0x00000002; guidePostsByteCount_ = input.readInt64(); break; } case 32: { - bitField0_ |= 0x00000008; + bitField0_ |= 0x00000004; keyBytesCount_ = input.readInt64(); break; } case 40: { - bitField0_ |= 0x00000010; + bitField0_ |= 0x00000008; guidePostsCount_ = input.readInt32(); break; } case 50: { org.apache.phoenix.coprocessor.generated.PGuidePostsProtos.PGuidePosts.Builder subBuilder = null; - if (((bitField0_ & 0x00000020) == 0x00000020)) { + if (((bitField0_ & 0x00000010) == 0x00000010)) { subBuilder = pGuidePosts_.toBuilder(); } pGuidePosts_ = input.readMessage(org.apache.phoenix.coprocessor.generated.PGuidePostsProtos.PGuidePosts.PARSER, extensionRegistry); @@ -1993,10 +2010,15 @@ private PTableStats( subBuilder.mergeFrom(pGuidePosts_); pGuidePosts_ = subBuilder.buildPartial(); } + bitField0_ |= 0x00000010; + break; + } + case 58: { bitField0_ |= 0x00000020; + guidePosts_ = input.readBytes(); break; } - case 56: { + case 64: { bitField0_ |= 0x00000040; maxLength_ = input.readInt32(); break; @@ -2009,6 +2031,9 @@ private PTableStats( throw new com.google.protobuf.InvalidProtocolBufferException( e.getMessage()).setUnfinishedMessage(this); } finally { + if (((mutable_bitField0_ & 0x00000002) == 0x00000002)) { + values_ = java.util.Collections.unmodifiableList(values_); + } this.unknownFields = unknownFields.build(); makeExtensionsImmutable(); } @@ -2057,20 +2082,27 @@ public com.google.protobuf.ByteString getKey() { return key_; } - // optional bytes guidePosts = 2; - public static final int GUIDEPOSTS_FIELD_NUMBER = 2; - private com.google.protobuf.ByteString guidePosts_; + // repeated bytes values = 2; + public static final int VALUES_FIELD_NUMBER = 2; + private java.util.List values_; /** - * optional bytes guidePosts = 2; + * repeated bytes values = 2; */ - public boolean hasGuidePosts() { - return ((bitField0_ & 0x00000002) == 0x00000002); + public java.util.List + getValuesList() { + return values_; } /** - * optional bytes guidePosts = 2; + * repeated bytes values = 2; */ - public com.google.protobuf.ByteString getGuidePosts() { - return guidePosts_; + public int getValuesCount() { + return values_.size(); + } + /** + * repeated bytes values = 2; + */ + public com.google.protobuf.ByteString getValues(int index) { + return values_.get(index); } // optional int64 guidePostsByteCount = 3; @@ -2080,7 +2112,7 @@ public com.google.protobuf.ByteString getGuidePosts() { * optional int64 guidePostsByteCount = 3; */ public boolean hasGuidePostsByteCount() { - return ((bitField0_ & 0x00000004) == 0x00000004); + return ((bitField0_ & 0x00000002) == 0x00000002); } /** * optional int64 guidePostsByteCount = 3; @@ -2096,7 +2128,7 @@ public long getGuidePostsByteCount() { * optional int64 keyBytesCount = 4; */ public boolean hasKeyBytesCount() { - return ((bitField0_ & 0x00000008) == 0x00000008); + return ((bitField0_ & 0x00000004) == 0x00000004); } /** * optional int64 keyBytesCount = 4; @@ -2112,7 +2144,7 @@ public long getKeyBytesCount() { * optional int32 guidePostsCount = 5; */ public boolean hasGuidePostsCount() { - return ((bitField0_ & 0x00000010) == 0x00000010); + return ((bitField0_ & 0x00000008) == 0x00000008); } /** * optional int32 guidePostsCount = 5; @@ -2128,7 +2160,7 @@ public int getGuidePostsCount() { * optional .PGuidePosts pGuidePosts = 6; */ public boolean hasPGuidePosts() { - return ((bitField0_ & 0x00000020) == 0x00000020); + return ((bitField0_ & 0x00000010) == 0x00000010); } /** * optional .PGuidePosts pGuidePosts = 6; @@ -2143,17 +2175,33 @@ public org.apache.phoenix.coprocessor.generated.PGuidePostsProtos.PGuidePostsOrB return pGuidePosts_; } - // optional int32 maxLength = 7; - public static final int MAXLENGTH_FIELD_NUMBER = 7; + // optional bytes guidePosts = 7; + public static final int GUIDEPOSTS_FIELD_NUMBER = 7; + private com.google.protobuf.ByteString guidePosts_; + /** + * optional bytes guidePosts = 7; + */ + public boolean hasGuidePosts() { + return ((bitField0_ & 0x00000020) == 0x00000020); + } + /** + * optional bytes guidePosts = 7; + */ + public com.google.protobuf.ByteString getGuidePosts() { + return guidePosts_; + } + + // optional int32 maxLength = 8; + public static final int MAXLENGTH_FIELD_NUMBER = 8; private int maxLength_; /** - * optional int32 maxLength = 7; + * optional int32 maxLength = 8; */ public boolean hasMaxLength() { return ((bitField0_ & 0x00000040) == 0x00000040); } /** - * optional int32 maxLength = 7; + * optional int32 maxLength = 8; */ public int getMaxLength() { return maxLength_; @@ -2161,11 +2209,12 @@ public int getMaxLength() { private void initFields() { key_ = com.google.protobuf.ByteString.EMPTY; - guidePosts_ = com.google.protobuf.ByteString.EMPTY; + values_ = java.util.Collections.emptyList(); guidePostsByteCount_ = 0L; keyBytesCount_ = 0L; guidePostsCount_ = 0; pGuidePosts_ = org.apache.phoenix.coprocessor.generated.PGuidePostsProtos.PGuidePosts.getDefaultInstance(); + guidePosts_ = com.google.protobuf.ByteString.EMPTY; maxLength_ = 0; } private byte memoizedIsInitialized = -1; @@ -2187,23 +2236,26 @@ public void writeTo(com.google.protobuf.CodedOutputStream output) if (((bitField0_ & 0x00000001) == 0x00000001)) { output.writeBytes(1, key_); } + for (int i = 0; i < values_.size(); i++) { + output.writeBytes(2, values_.get(i)); + } if (((bitField0_ & 0x00000002) == 0x00000002)) { - output.writeBytes(2, guidePosts_); + output.writeInt64(3, guidePostsByteCount_); } if (((bitField0_ & 0x00000004) == 0x00000004)) { - output.writeInt64(3, guidePostsByteCount_); + output.writeInt64(4, keyBytesCount_); } if (((bitField0_ & 0x00000008) == 0x00000008)) { - output.writeInt64(4, keyBytesCount_); + output.writeInt32(5, guidePostsCount_); } if (((bitField0_ & 0x00000010) == 0x00000010)) { - output.writeInt32(5, guidePostsCount_); + output.writeMessage(6, pGuidePosts_); } if (((bitField0_ & 0x00000020) == 0x00000020)) { - output.writeMessage(6, pGuidePosts_); + output.writeBytes(7, guidePosts_); } if (((bitField0_ & 0x00000040) == 0x00000040)) { - output.writeInt32(7, maxLength_); + output.writeInt32(8, maxLength_); } getUnknownFields().writeTo(output); } @@ -2218,29 +2270,38 @@ public int getSerializedSize() { size += com.google.protobuf.CodedOutputStream .computeBytesSize(1, key_); } + { + int dataSize = 0; + for (int i = 0; i < values_.size(); i++) { + dataSize += com.google.protobuf.CodedOutputStream + .computeBytesSizeNoTag(values_.get(i)); + } + size += dataSize; + size += 1 * getValuesList().size(); + } if (((bitField0_ & 0x00000002) == 0x00000002)) { size += com.google.protobuf.CodedOutputStream - .computeBytesSize(2, guidePosts_); + .computeInt64Size(3, guidePostsByteCount_); } if (((bitField0_ & 0x00000004) == 0x00000004)) { size += com.google.protobuf.CodedOutputStream - .computeInt64Size(3, guidePostsByteCount_); + .computeInt64Size(4, keyBytesCount_); } if (((bitField0_ & 0x00000008) == 0x00000008)) { size += com.google.protobuf.CodedOutputStream - .computeInt64Size(4, keyBytesCount_); + .computeInt32Size(5, guidePostsCount_); } if (((bitField0_ & 0x00000010) == 0x00000010)) { size += com.google.protobuf.CodedOutputStream - .computeInt32Size(5, guidePostsCount_); + .computeMessageSize(6, pGuidePosts_); } if (((bitField0_ & 0x00000020) == 0x00000020)) { size += com.google.protobuf.CodedOutputStream - .computeMessageSize(6, pGuidePosts_); + .computeBytesSize(7, guidePosts_); } if (((bitField0_ & 0x00000040) == 0x00000040)) { size += com.google.protobuf.CodedOutputStream - .computeInt32Size(7, maxLength_); + .computeInt32Size(8, maxLength_); } size += getUnknownFields().getSerializedSize(); memoizedSerializedSize = size; @@ -2270,11 +2331,8 @@ public boolean equals(final java.lang.Object obj) { result = result && getKey() .equals(other.getKey()); } - result = result && (hasGuidePosts() == other.hasGuidePosts()); - if (hasGuidePosts()) { - result = result && getGuidePosts() - .equals(other.getGuidePosts()); - } + result = result && getValuesList() + .equals(other.getValuesList()); result = result && (hasGuidePostsByteCount() == other.hasGuidePostsByteCount()); if (hasGuidePostsByteCount()) { result = result && (getGuidePostsByteCount() @@ -2295,6 +2353,11 @@ public boolean equals(final java.lang.Object obj) { result = result && getPGuidePosts() .equals(other.getPGuidePosts()); } + result = result && (hasGuidePosts() == other.hasGuidePosts()); + if (hasGuidePosts()) { + result = result && getGuidePosts() + .equals(other.getGuidePosts()); + } result = result && (hasMaxLength() == other.hasMaxLength()); if (hasMaxLength()) { result = result && (getMaxLength() @@ -2317,9 +2380,9 @@ public int hashCode() { hash = (37 * hash) + KEY_FIELD_NUMBER; hash = (53 * hash) + getKey().hashCode(); } - if (hasGuidePosts()) { - hash = (37 * hash) + GUIDEPOSTS_FIELD_NUMBER; - hash = (53 * hash) + getGuidePosts().hashCode(); + if (getValuesCount() > 0) { + hash = (37 * hash) + VALUES_FIELD_NUMBER; + hash = (53 * hash) + getValuesList().hashCode(); } if (hasGuidePostsByteCount()) { hash = (37 * hash) + GUIDEPOSTSBYTECOUNT_FIELD_NUMBER; @@ -2337,6 +2400,10 @@ public int hashCode() { hash = (37 * hash) + PGUIDEPOSTS_FIELD_NUMBER; hash = (53 * hash) + getPGuidePosts().hashCode(); } + if (hasGuidePosts()) { + hash = (37 * hash) + GUIDEPOSTS_FIELD_NUMBER; + hash = (53 * hash) + getGuidePosts().hashCode(); + } if (hasMaxLength()) { hash = (37 * hash) + MAXLENGTH_FIELD_NUMBER; hash = (53 * hash) + getMaxLength(); @@ -2453,7 +2520,7 @@ public Builder clear() { super.clear(); key_ = com.google.protobuf.ByteString.EMPTY; bitField0_ = (bitField0_ & ~0x00000001); - guidePosts_ = com.google.protobuf.ByteString.EMPTY; + values_ = java.util.Collections.emptyList(); bitField0_ = (bitField0_ & ~0x00000002); guidePostsByteCount_ = 0L; bitField0_ = (bitField0_ & ~0x00000004); @@ -2467,8 +2534,10 @@ public Builder clear() { pGuidePostsBuilder_.clear(); } bitField0_ = (bitField0_ & ~0x00000020); - maxLength_ = 0; + guidePosts_ = com.google.protobuf.ByteString.EMPTY; bitField0_ = (bitField0_ & ~0x00000040); + maxLength_ = 0; + bitField0_ = (bitField0_ & ~0x00000080); return this; } @@ -2501,24 +2570,25 @@ public org.apache.phoenix.coprocessor.generated.PTableProtos.PTableStats buildPa to_bitField0_ |= 0x00000001; } result.key_ = key_; - if (((from_bitField0_ & 0x00000002) == 0x00000002)) { - to_bitField0_ |= 0x00000002; + if (((bitField0_ & 0x00000002) == 0x00000002)) { + values_ = java.util.Collections.unmodifiableList(values_); + bitField0_ = (bitField0_ & ~0x00000002); } - result.guidePosts_ = guidePosts_; + result.values_ = values_; if (((from_bitField0_ & 0x00000004) == 0x00000004)) { - to_bitField0_ |= 0x00000004; + to_bitField0_ |= 0x00000002; } result.guidePostsByteCount_ = guidePostsByteCount_; if (((from_bitField0_ & 0x00000008) == 0x00000008)) { - to_bitField0_ |= 0x00000008; + to_bitField0_ |= 0x00000004; } result.keyBytesCount_ = keyBytesCount_; if (((from_bitField0_ & 0x00000010) == 0x00000010)) { - to_bitField0_ |= 0x00000010; + to_bitField0_ |= 0x00000008; } result.guidePostsCount_ = guidePostsCount_; if (((from_bitField0_ & 0x00000020) == 0x00000020)) { - to_bitField0_ |= 0x00000020; + to_bitField0_ |= 0x00000010; } if (pGuidePostsBuilder_ == null) { result.pGuidePosts_ = pGuidePosts_; @@ -2526,6 +2596,10 @@ public org.apache.phoenix.coprocessor.generated.PTableProtos.PTableStats buildPa result.pGuidePosts_ = pGuidePostsBuilder_.build(); } if (((from_bitField0_ & 0x00000040) == 0x00000040)) { + to_bitField0_ |= 0x00000020; + } + result.guidePosts_ = guidePosts_; + if (((from_bitField0_ & 0x00000080) == 0x00000080)) { to_bitField0_ |= 0x00000040; } result.maxLength_ = maxLength_; @@ -2548,8 +2622,15 @@ public Builder mergeFrom(org.apache.phoenix.coprocessor.generated.PTableProtos.P if (other.hasKey()) { setKey(other.getKey()); } - if (other.hasGuidePosts()) { - setGuidePosts(other.getGuidePosts()); + if (!other.values_.isEmpty()) { + if (values_.isEmpty()) { + values_ = other.values_; + bitField0_ = (bitField0_ & ~0x00000002); + } else { + ensureValuesIsMutable(); + values_.addAll(other.values_); + } + onChanged(); } if (other.hasGuidePostsByteCount()) { setGuidePostsByteCount(other.getGuidePostsByteCount()); @@ -2563,6 +2644,9 @@ public Builder mergeFrom(org.apache.phoenix.coprocessor.generated.PTableProtos.P if (other.hasPGuidePosts()) { mergePGuidePosts(other.getPGuidePosts()); } + if (other.hasGuidePosts()) { + setGuidePosts(other.getGuidePosts()); + } if (other.hasMaxLength()) { setMaxLength(other.getMaxLength()); } @@ -2633,38 +2717,74 @@ public Builder clearKey() { return this; } - // optional bytes guidePosts = 2; - private com.google.protobuf.ByteString guidePosts_ = com.google.protobuf.ByteString.EMPTY; + // repeated bytes values = 2; + private java.util.List values_ = java.util.Collections.emptyList(); + private void ensureValuesIsMutable() { + if (!((bitField0_ & 0x00000002) == 0x00000002)) { + values_ = new java.util.ArrayList(values_); + bitField0_ |= 0x00000002; + } + } /** - * optional bytes guidePosts = 2; + * repeated bytes values = 2; */ - public boolean hasGuidePosts() { - return ((bitField0_ & 0x00000002) == 0x00000002); + public java.util.List + getValuesList() { + return java.util.Collections.unmodifiableList(values_); } /** - * optional bytes guidePosts = 2; + * repeated bytes values = 2; */ - public com.google.protobuf.ByteString getGuidePosts() { - return guidePosts_; + public int getValuesCount() { + return values_.size(); } /** - * optional bytes guidePosts = 2; + * repeated bytes values = 2; */ - public Builder setGuidePosts(com.google.protobuf.ByteString value) { + public com.google.protobuf.ByteString getValues(int index) { + return values_.get(index); + } + /** + * repeated bytes values = 2; + */ + public Builder setValues( + int index, com.google.protobuf.ByteString value) { if (value == null) { throw new NullPointerException(); } - bitField0_ |= 0x00000002; - guidePosts_ = value; + ensureValuesIsMutable(); + values_.set(index, value); onChanged(); return this; } /** - * optional bytes guidePosts = 2; + * repeated bytes values = 2; */ - public Builder clearGuidePosts() { + public Builder addValues(com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + ensureValuesIsMutable(); + values_.add(value); + onChanged(); + return this; + } + /** + * repeated bytes values = 2; + */ + public Builder addAllValues( + java.lang.Iterable values) { + ensureValuesIsMutable(); + super.addAll(values, values_); + onChanged(); + return this; + } + /** + * repeated bytes values = 2; + */ + public Builder clearValues() { + values_ = java.util.Collections.emptyList(); bitField0_ = (bitField0_ & ~0x00000002); - guidePosts_ = getDefaultInstance().getGuidePosts(); onChanged(); return this; } @@ -2885,34 +3005,70 @@ public org.apache.phoenix.coprocessor.generated.PGuidePostsProtos.PGuidePostsOrB return pGuidePostsBuilder_; } - // optional int32 maxLength = 7; + // optional bytes guidePosts = 7; + private com.google.protobuf.ByteString guidePosts_ = com.google.protobuf.ByteString.EMPTY; + /** + * optional bytes guidePosts = 7; + */ + public boolean hasGuidePosts() { + return ((bitField0_ & 0x00000040) == 0x00000040); + } + /** + * optional bytes guidePosts = 7; + */ + public com.google.protobuf.ByteString getGuidePosts() { + return guidePosts_; + } + /** + * optional bytes guidePosts = 7; + */ + public Builder setGuidePosts(com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000040; + guidePosts_ = value; + onChanged(); + return this; + } + /** + * optional bytes guidePosts = 7; + */ + public Builder clearGuidePosts() { + bitField0_ = (bitField0_ & ~0x00000040); + guidePosts_ = getDefaultInstance().getGuidePosts(); + onChanged(); + return this; + } + + // optional int32 maxLength = 8; private int maxLength_ ; /** - * optional int32 maxLength = 7; + * optional int32 maxLength = 8; */ public boolean hasMaxLength() { - return ((bitField0_ & 0x00000040) == 0x00000040); + return ((bitField0_ & 0x00000080) == 0x00000080); } /** - * optional int32 maxLength = 7; + * optional int32 maxLength = 8; */ public int getMaxLength() { return maxLength_; } /** - * optional int32 maxLength = 7; + * optional int32 maxLength = 8; */ public Builder setMaxLength(int value) { - bitField0_ |= 0x00000040; + bitField0_ |= 0x00000080; maxLength_ = value; onChanged(); return this; } /** - * optional int32 maxLength = 7; + * optional int32 maxLength = 8; */ public Builder clearMaxLength() { - bitField0_ = (bitField0_ & ~0x00000040); + bitField0_ = (bitField0_ & ~0x00000080); maxLength_ = 0; onChanged(); return this; @@ -6853,31 +7009,32 @@ public Builder clearTransactional() { "\006 \002(\010\022\020\n\010position\030\007 \002(\005\022\021\n\tsortOrder\030\010 \002" + "(\005\022\021\n\tarraySize\030\t \001(\005\022\024\n\014viewConstant\030\n " + "\001(\014\022\026\n\016viewReferenced\030\013 \001(\010\022\022\n\nexpressio" + - "n\030\014 \001(\t\022\026\n\016isRowTimestamp\030\r \001(\010\"\261\001\n\013PTab" + - "leStats\022\013\n\003key\030\001 \002(\014\022\022\n\nguidePosts\030\002 \001(\014" + - "\022\033\n\023guidePostsByteCount\030\003 \001(\003\022\025\n\rkeyByte", - "sCount\030\004 \001(\003\022\027\n\017guidePostsCount\030\005 \001(\005\022!\n" + - "\013pGuidePosts\030\006 \001(\0132\014.PGuidePosts\022\021\n\tmaxL" + - "ength\030\007 \001(\005\"\206\005\n\006PTable\022\027\n\017schemaNameByte" + - "s\030\001 \002(\014\022\026\n\016tableNameBytes\030\002 \002(\014\022\036\n\ttable" + - "Type\030\003 \002(\0162\013.PTableType\022\022\n\nindexState\030\004 " + - "\001(\t\022\026\n\016sequenceNumber\030\005 \002(\003\022\021\n\ttimeStamp" + - "\030\006 \002(\003\022\023\n\013pkNameBytes\030\007 \001(\014\022\021\n\tbucketNum" + - "\030\010 \002(\005\022\031\n\007columns\030\t \003(\0132\010.PColumn\022\030\n\007ind" + - "exes\030\n \003(\0132\007.PTable\022\027\n\017isImmutableRows\030\013" + - " \002(\010\022 \n\nguidePosts\030\014 \003(\0132\014.PTableStats\022\032", - "\n\022dataTableNameBytes\030\r \001(\014\022\031\n\021defaultFam" + - "ilyName\030\016 \001(\014\022\022\n\ndisableWAL\030\017 \002(\010\022\023\n\013mul" + - "tiTenant\030\020 \002(\010\022\020\n\010viewType\030\021 \001(\014\022\025\n\rview" + - "Statement\030\022 \001(\014\022\025\n\rphysicalNames\030\023 \003(\014\022\020" + - "\n\010tenantId\030\024 \001(\014\022\023\n\013viewIndexId\030\025 \001(\005\022\021\n" + - "\tindexType\030\026 \001(\014\022\026\n\016statsTimeStamp\030\027 \001(\003" + - "\022\022\n\nstoreNulls\030\030 \001(\010\022\027\n\017baseColumnCount\030" + - "\031 \001(\005\022\036\n\026rowKeyOrderOptimizable\030\032 \001(\010\022\025\n" + - "\rtransactional\030\033 \001(\010*A\n\nPTableType\022\n\n\006SY" + - "STEM\020\000\022\010\n\004USER\020\001\022\010\n\004VIEW\020\002\022\t\n\005INDEX\020\003\022\010\n", - "\004JOIN\020\004B@\n(org.apache.phoenix.coprocesso" + - "r.generatedB\014PTableProtosH\001\210\001\001\240\001\001" + "n\030\014 \001(\t\022\026\n\016isRowTimestamp\030\r \001(\010\"\301\001\n\013PTab" + + "leStats\022\013\n\003key\030\001 \002(\014\022\016\n\006values\030\002 \003(\014\022\033\n\023" + + "guidePostsByteCount\030\003 \001(\003\022\025\n\rkeyBytesCou", + "nt\030\004 \001(\003\022\027\n\017guidePostsCount\030\005 \001(\005\022!\n\013pGu" + + "idePosts\030\006 \001(\0132\014.PGuidePosts\022\022\n\nguidePos" + + "ts\030\007 \001(\014\022\021\n\tmaxLength\030\010 \001(\005\"\206\005\n\006PTable\022\027" + + "\n\017schemaNameBytes\030\001 \002(\014\022\026\n\016tableNameByte" + + "s\030\002 \002(\014\022\036\n\ttableType\030\003 \002(\0162\013.PTableType\022" + + "\022\n\nindexState\030\004 \001(\t\022\026\n\016sequenceNumber\030\005 " + + "\002(\003\022\021\n\ttimeStamp\030\006 \002(\003\022\023\n\013pkNameBytes\030\007 " + + "\001(\014\022\021\n\tbucketNum\030\010 \002(\005\022\031\n\007columns\030\t \003(\0132" + + "\010.PColumn\022\030\n\007indexes\030\n \003(\0132\007.PTable\022\027\n\017i" + + "sImmutableRows\030\013 \002(\010\022 \n\nguidePosts\030\014 \003(\013", + "2\014.PTableStats\022\032\n\022dataTableNameBytes\030\r \001" + + "(\014\022\031\n\021defaultFamilyName\030\016 \001(\014\022\022\n\ndisable" + + "WAL\030\017 \002(\010\022\023\n\013multiTenant\030\020 \002(\010\022\020\n\010viewTy" + + "pe\030\021 \001(\014\022\025\n\rviewStatement\030\022 \001(\014\022\025\n\rphysi" + + "calNames\030\023 \003(\014\022\020\n\010tenantId\030\024 \001(\014\022\023\n\013view" + + "IndexId\030\025 \001(\005\022\021\n\tindexType\030\026 \001(\014\022\026\n\016stat" + + "sTimeStamp\030\027 \001(\003\022\022\n\nstoreNulls\030\030 \001(\010\022\027\n\017" + + "baseColumnCount\030\031 \001(\005\022\036\n\026rowKeyOrderOpti" + + "mizable\030\032 \001(\010\022\025\n\rtransactional\030\033 \001(\010*A\n\n" + + "PTableType\022\n\n\006SYSTEM\020\000\022\010\n\004USER\020\001\022\010\n\004VIEW", + "\020\002\022\t\n\005INDEX\020\003\022\010\n\004JOIN\020\004B@\n(org.apache.ph" + + "oenix.coprocessor.generatedB\014PTableProto" + + "sH\001\210\001\001\240\001\001" }; com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner = new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() { @@ -6895,7 +7052,7 @@ public com.google.protobuf.ExtensionRegistry assignDescriptors( internal_static_PTableStats_fieldAccessorTable = new com.google.protobuf.GeneratedMessage.FieldAccessorTable( internal_static_PTableStats_descriptor, - new java.lang.String[] { "Key", "GuidePosts", "GuidePostsByteCount", "KeyBytesCount", "GuidePostsCount", "PGuidePosts", "MaxLength", }); + new java.lang.String[] { "Key", "Values", "GuidePostsByteCount", "KeyBytesCount", "GuidePostsCount", "PGuidePosts", "GuidePosts", "MaxLength", }); internal_static_PTable_descriptor = getDescriptor().getMessageTypes().get(2); internal_static_PTable_fieldAccessorTable = new diff --git a/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java b/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java index 8977217bbc1..d1300b54429 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java @@ -464,9 +464,6 @@ private List> getParallelScans(byte[] startKey, byte[] stopKey) throw whereConditions.add(where.getFirst()); } GuidePostsInfo gps = getGuidePosts(whereConditions); - if (logger.isDebugEnabled()) { - logger.debug("Guideposts: " + gps); - } boolean traverseAllRegions = isSalted || isLocalIndex; if (!traverseAllRegions) { byte[] scanStartRow = scan.getStartRow(); @@ -492,7 +489,7 @@ private List> getParallelScans(byte[] startKey, byte[] stopKey) throw } List> parallelScans = Lists.newArrayListWithExpectedSize(stopIndex - regionIndex + 1); - ImmutableBytesWritable currentKey = new ImmutableBytesWritable(startKey, 0, startKey.length); + ImmutableBytesWritable currentKey = new ImmutableBytesWritable(startKey); int gpsSize = gps.getGuidePostsCount(); int estGuidepostsPerRegion = gpsSize == 0 ? 1 : gpsSize / regionLocations.size() + 1; diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsInfo.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsInfo.java index b7f5c49f87c..da7d3a51358 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsInfo.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsInfo.java @@ -17,15 +17,8 @@ */ package org.apache.phoenix.schema.stats; -import java.io.DataOutputStream; -import java.io.IOException; - import org.apache.hadoop.hbase.io.ImmutableBytesWritable; -import org.apache.hadoop.hbase.util.Bytes; import org.apache.phoenix.util.ByteUtil; -import org.apache.phoenix.util.CodecUtils; -import org.apache.phoenix.util.PrefixByteEncoder; -import org.apache.phoenix.util.TrustedByteArrayOutputStream; /** * A class that holds the guidePosts of a region and also allows combining the * guidePosts of different regions when the GuidePostsInfo is formed for a table. @@ -57,13 +50,8 @@ public class GuidePostsInfo { public int getMaxLength() { return maxLength; } - - private TrustedByteArrayOutputStream stream; - private PrefixByteEncoder encoder; - private DataOutputStream output; - private byte[] lastRow; + private int guidePostsCount; - private boolean isStreamInitialized; /** * Constructor that creates GuidePostsInfo per region @@ -77,7 +65,6 @@ public GuidePostsInfo(long byteCount, ImmutableBytesWritable guidePosts, long ro this.byteCount = byteCount; this.rowCount = rowCount; this.guidePostsCount = guidePostsCount; - lastRow = ByteUtil.EMPTY_BYTE_ARRAY; } @@ -93,69 +80,8 @@ public long getRowCount() { return this.rowCount; } - public void incrementRowCount() { - this.rowCount++; - } - public int getGuidePostsCount() { return guidePostsCount; } - /** - * The guide posts, rowCount and byteCount are accumulated every time a guidePosts depth is - * reached while collecting stats. - * @param row - * @param byteCount - * @return - * @throws IOException - */ - public boolean encodeAndCollectGuidePost(byte[] row, long byteCount, long rowCount) { - if (row.length != 0 && Bytes.compareTo(lastRow, row) < 0) { - try { - if(!isStreamInitialized){ - stream = new TrustedByteArrayOutputStream(guidePosts.getLength()); - output = new DataOutputStream(stream); - stream.write(ByteUtil.copyKeyBytesIfNecessary(guidePosts)); - encoder = new PrefixByteEncoder(); - isStreamInitialized=true; - } - encoder.encode(output, row, 0, row.length); - this.byteCount += byteCount; - this.guidePostsCount++; - this.maxLength = encoder.getMaxLength(); - this.rowCount += rowCount; - lastRow = row; - return true; - } catch (IOException e) { - return false; - } - } - return false; - } - - public boolean encodeAndCollectGuidePost(byte[] row){ - return encodeAndCollectGuidePost(row, 0, 0); - } - - public boolean encodeAndCollectGuidePost(byte[] row, long byteCount){ - return encodeAndCollectGuidePost(row, byteCount, 0); - } - - public void close() { - CodecUtils.close(stream); - } - - /* - * This needs to be collect once all stats are encoded and stream buffer needs to be copied for retrieval before close() - */ - public void updateGuidePosts() { - if (stream != null) { - this.guidePosts.set(Bytes.copy(stream.getBuffer(), 0, stream.size())); - } - } - - @Override - public String toString() { - return Bytes.toString(this.guidePosts.get()); - } } \ No newline at end of file diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsInfoWriter.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsInfoWriter.java new file mode 100644 index 00000000000..f0963e0a970 --- /dev/null +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsInfoWriter.java @@ -0,0 +1,122 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.schema.stats; + +import java.io.DataOutputStream; +import java.io.IOException; + +import org.apache.hadoop.hbase.io.ImmutableBytesWritable; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.phoenix.util.ByteUtil; +import org.apache.phoenix.util.CodecUtils; +import org.apache.phoenix.util.PrefixByteEncoder; +import org.apache.phoenix.util.TrustedByteArrayOutputStream; + +/* + * Writer to help in writing guidePosts and creating guidePostInfo. This is used when we are collecting stats or reading stats for a table. + * updateGuidePosts() is required to be invoked before calling getGuidePostInfo() in order to get the updated guidePosts + */ + +public class GuidePostsInfoWriter { + private PrefixByteEncoder encoder; + private byte[] lastRow; + private GuidePostsInfo guidePostInfo; + private long byteCount = 0; + private int guidePostsCount; + + /** + * The rowCount that is flattened across the total number of guide posts. + */ + private long rowCount = 0; + + /** + * Maximum length of a guidePost collected + */ + private int maxLength; + private DataOutputStream output; + private TrustedByteArrayOutputStream stream; + private ImmutableBytesWritable guidePosts=new ImmutableBytesWritable(); + + public final static GuidePostsInfo EMPTY_GUIDEPOST = new GuidePostsInfo(0, + new ImmutableBytesWritable(ByteUtil.EMPTY_BYTE_ARRAY), 0, 0, 0); + + public int getMaxLength() { + return maxLength; + } + public GuidePostsInfoWriter(){ + this.stream = new TrustedByteArrayOutputStream(1); + this.output = new DataOutputStream(stream); + this.encoder=new PrefixByteEncoder(); + lastRow = ByteUtil.EMPTY_BYTE_ARRAY; + } + + /** + * The guide posts, rowCount and byteCount are accumulated every time a guidePosts depth is + * reached while collecting stats. + * @param row + * @param byteCount + * @return + * @throws IOException + */ + public boolean writeGuidePosts( byte[] row, long byteCount, long rowCount) { + if (row.length != 0 && Bytes.compareTo(lastRow, row) < 0) { + try { + encoder.encode(output, row, 0, row.length); + this.byteCount += byteCount; + this.guidePostsCount++; + this.maxLength = encoder.getMaxLength(); + this.rowCount += rowCount; + lastRow = row; + return true; + } catch (IOException e) { + return false; + } + } + return false; + } + + public boolean writeGuidePosts(byte[] row){ + return writeGuidePosts(row, 0, 0); + } + + public boolean writeGuidePosts(byte[] row, long byteCount){ + return writeGuidePosts(row, byteCount, 0); + } + + /* + * This needs to be called once all stats are encoded and stream buffer needs to be copied for retrieval before + * close() + */ + public void updateGuidePosts() { + this.guidePosts.set(Bytes.copy(stream.getBuffer(), 0, stream.size())); + this.guidePostInfo = new GuidePostsInfo(this.byteCount, this.guidePosts, this.rowCount, this.maxLength, + this.guidePostsCount); + } + + public GuidePostsInfo getGuidePostInfo() { + return this.guidePostInfo; + } + + public void close() { + CodecUtils.close(stream); + } + + public void incrementRowCount() { + this.rowCount++; + } +} diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollector.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollector.java index e51e2bdca4d..0c619640541 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollector.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollector.java @@ -60,9 +60,9 @@ public class StatisticsCollector { private long guidepostDepth; private long maxTimeStamp = MetaDataProtocol.MIN_TABLE_TIMESTAMP; - private Map> guidePostsMap = Maps.newHashMap(); + private Map> guidePostsInfoWriterMap = Maps.newHashMap(); protected StatisticsWriter statsTable; - private Pair cachedGps = null; + private Pair cachedGps = null; public StatisticsCollector(RegionCoprocessorEnvironment env, String tableName, long clientTimeStamp) throws IOException { @@ -98,8 +98,8 @@ public StatisticsCollector(RegionCoprocessorEnvironment env, String tableName, l // in a compaction we know the one family ahead of time if (family != null) { ImmutableBytesPtr cfKey = new ImmutableBytesPtr(family); - cachedGps = new Pair(0l, new GuidePostsInfo(0, new ImmutableBytesWritable(ByteUtil.EMPTY_BYTE_ARRAY), 0l, 0, 0)); - guidePostsMap.put(cfKey, cachedGps); + cachedGps = new Pair(0l, new GuidePostsInfoWriter()); + guidePostsInfoWriterMap.put(cfKey, cachedGps); } } @@ -109,10 +109,10 @@ public long getMaxTimeStamp() { public void close() throws IOException { this.statsTable.close(); - for (ImmutableBytesPtr ptr : guidePostsMap.keySet()) { - GuidePostsInfo guidePostInfo = guidePostsMap.get(ptr).getSecond(); - if (guidePostInfo != null) { - guidePostInfo.close(); + for (ImmutableBytesPtr ptr : guidePostsInfoWriterMap.keySet()) { + GuidePostsInfoWriter guidePostWriter = guidePostsInfoWriterMap.get(ptr).getSecond(); + if (guidePostWriter != null) { + guidePostWriter.close(); } } } @@ -136,7 +136,7 @@ private void writeStatsToStatsTable(final Region region, boolean delete, List mutations) throws IOException { */ public void collectStatistics(final List results) { Map famMap = Maps.newHashMap(); - List rowTracker = null; + List rowTracker = null; if (cachedGps == null) { - rowTracker = new ArrayList(); + rowTracker = new ArrayList(); } for (Cell cell : results) { KeyValue kv = KeyValueUtil.ensureKeyValue(cell); maxTimeStamp = Math.max(maxTimeStamp, kv.getTimestamp()); - Pair gps; + Pair gps; if (cachedGps == null) { ImmutableBytesPtr cfKey = new ImmutableBytesPtr(kv.getFamilyArray(), kv.getFamilyOffset(), kv.getFamilyLength()); - gps = guidePostsMap.get(cfKey); + gps = guidePostsInfoWriterMap.get(cfKey); if (gps == null) { - gps = new Pair(0l, - new GuidePostsInfo(0,new ImmutableBytesWritable(ByteUtil.EMPTY_BYTE_ARRAY), 0l, 0, 0)); - guidePostsMap.put(cfKey, gps); + gps = new Pair(0l, + new GuidePostsInfoWriter()); + guidePostsInfoWriterMap.put(cfKey, gps); } if (famMap.get(cfKey) == null) { famMap.put(cfKey, true); @@ -196,13 +196,13 @@ public void collectStatistics(final List results) { if (byteCount >= guidepostDepth) { byte[] row = ByteUtil.copyKeyBytesIfNecessary( new ImmutableBytesWritable(kv.getRowArray(), kv.getRowOffset(), kv.getRowLength())); - if (gps.getSecond().encodeAndCollectGuidePost(row, byteCount)) { + if (gps.getSecond().writeGuidePosts(row, byteCount)) { gps.setFirst(0l); } } } if (cachedGps == null) { - for (GuidePostsInfo s : rowTracker) { + for (GuidePostsInfoWriter s : rowTracker) { s.incrementRowCount(); } } else { @@ -226,21 +226,21 @@ protected InternalScanner getInternalScanner(RegionCoprocessorEnvironment env, S } public void clear() { - this.guidePostsMap.clear(); + this.guidePostsInfoWriterMap.clear(); maxTimeStamp = MetaDataProtocol.MIN_TABLE_TIMESTAMP; } public GuidePostsInfo getGuidePosts(ImmutableBytesPtr fam) { - Pair pair = guidePostsMap.get(fam); - if (pair != null) { return pair.getSecond(); } + Pair pair = guidePostsInfoWriterMap.get(fam); + if (pair != null) { return pair.getSecond().getGuidePostInfo(); } return null; } public void updateGuidePosts() { - for (ImmutableBytesPtr cf : guidePostsMap.keySet()) { - GuidePostsInfo guidePostInfo = guidePostsMap.get(cf).getSecond(); - if (guidePostInfo != null) { - guidePostInfo.updateGuidePosts(); + for (ImmutableBytesPtr cf : guidePostsInfoWriterMap.keySet()) { + GuidePostsInfoWriter guidePostWriter = guidePostsInfoWriterMap.get(cf).getSecond(); + if (guidePostWriter != null) { + guidePostWriter.updateGuidePosts(); } } } diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsUtil.java index 5ea7462d140..a1c63d71ce3 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsUtil.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsUtil.java @@ -22,6 +22,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.SortedMap; import java.util.TreeMap; import org.apache.hadoop.hbase.Cell; @@ -129,7 +130,7 @@ public static PTableStats readStatistics(HTableInterface statsHTable, byte[] tab s.addColumn(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES, QueryConstants.EMPTY_COLUMN_BYTES); ResultScanner scanner = null; long timeStamp = MetaDataProtocol.MIN_TABLE_TIMESTAMP; - TreeMap guidePostsPerCf = new TreeMap(Bytes.BYTES_COMPARATOR); + TreeMap guidePostsInfoWriterPerCf = new TreeMap(Bytes.BYTES_COMPARATOR); try { scanner = statsHTable.getScanner(s); Result result = null; @@ -171,36 +172,44 @@ public static PTableStats readStatistics(HTableInterface statsHTable, byte[] tab } if (cfName != null) { byte[] newGPStartKey = getGuidePostsInfoFromRowKey(tableNameBytes, cfName, result.getRow()); - GuidePostsInfo guidePosts = guidePostsPerCf.get(cfName); - if (guidePosts == null) { - guidePosts = new GuidePostsInfo(0l, new ImmutableBytesWritable(ByteUtil.EMPTY_BYTE_ARRAY), 0l, - 0, 0); - guidePostsPerCf.put(cfName, guidePosts); + GuidePostsInfoWriter guidePostsInfoWriter = guidePostsInfoWriterPerCf.get(cfName); + if (guidePostsInfoWriter == null) { + guidePostsInfoWriter = new GuidePostsInfoWriter(); + guidePostsInfoWriterPerCf.put(cfName, guidePostsInfoWriter); } - guidePosts.encodeAndCollectGuidePost(newGPStartKey, byteCount, rowCount); + guidePostsInfoWriter.writeGuidePosts(newGPStartKey, byteCount, rowCount); } } - for (byte[] cf : guidePostsPerCf.keySet()) { - GuidePostsInfo guidePostInfo = guidePostsPerCf.get(cf); - if (guidePostInfo != null) { - guidePostInfo.updateGuidePosts(); + for (byte[] cf : guidePostsInfoWriterPerCf.keySet()) { + GuidePostsInfoWriter guidePostsInfoWriter = guidePostsInfoWriterPerCf.get(cf); + if (guidePostsInfoWriter != null) { + guidePostsInfoWriter.updateGuidePosts(); } } } finally { if (scanner != null) { scanner.close(); } - for (byte[] cf : guidePostsPerCf.keySet()) { - GuidePostsInfo guidePostInfo = guidePostsPerCf.get(cf); - if (guidePostInfo != null) { - guidePostInfo.close(); + for (byte[] cf : guidePostsInfoWriterPerCf.keySet()) { + GuidePostsInfoWriter guidePostsInfoWriter = guidePostsInfoWriterPerCf.get(cf); + if (guidePostsInfoWriter != null) { + guidePostsInfoWriter.close(); } } } - if (!guidePostsPerCf.isEmpty()) { return new PTableStatsImpl(guidePostsPerCf, timeStamp); } + if (!guidePostsInfoWriterPerCf.isEmpty()) { return new PTableStatsImpl(getGuidePostsPerCf(guidePostsInfoWriterPerCf), timeStamp); } return PTableStats.EMPTY_STATS; } + private static SortedMap getGuidePostsPerCf( + TreeMap guidePostsWriterPerCf) { + TreeMap guidePostsPerCf = new TreeMap(Bytes.BYTES_COMPARATOR); + for (byte[] key : guidePostsWriterPerCf.keySet()) { + guidePostsPerCf.put(key, guidePostsWriterPerCf.get(key).getGuidePostInfo()); + } + return guidePostsPerCf; + } + public static long getGuidePostDepth(int guidepostPerRegion, long guidepostWidth, HTableDescriptor tableDesc) { if (guidepostPerRegion > 0) { long maxFileSize = HConstants.DEFAULT_MAX_FILE_SIZE; diff --git a/phoenix-protocol/src/main/PTable.proto b/phoenix-protocol/src/main/PTable.proto index 228c03c871b..59f34449141 100644 --- a/phoenix-protocol/src/main/PTable.proto +++ b/phoenix-protocol/src/main/PTable.proto @@ -52,12 +52,13 @@ message PColumn { message PTableStats { required bytes key = 1; - optional bytes guidePosts = 2; + repeated bytes values = 2; optional int64 guidePostsByteCount = 3; optional int64 keyBytesCount = 4; optional int32 guidePostsCount = 5; optional PGuidePosts pGuidePosts = 6; - optional int32 maxLength = 7; + optional bytes guidePosts = 7; + optional int32 maxLength = 8; } message PTable { From 538d9faefcf0282894288850ab8a5b4a0771c4b2 Mon Sep 17 00:00:00 2001 From: Ankit Singhal Date: Sat, 16 Jan 2016 21:55:02 +0530 Subject: [PATCH 09/47] Writer modified --- .../UngroupedAggregateRegionObserver.java | 1 - .../schema/stats/GuidePostsInfoWriter.java | 36 ++++++++++--------- .../schema/stats/StatisticsCollector.java | 11 +----- .../schema/stats/StatisticsScanner.java | 1 - .../phoenix/schema/stats/StatisticsUtil.java | 16 ++++----- 5 files changed, 28 insertions(+), 37 deletions(-) diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java index 7fb9ff65c74..1d67994747e 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java @@ -783,7 +783,6 @@ private long collectStatsInternal() throws IOException { } finally { try { if (noErrors && !compactionRunning) { - stats.updateGuidePosts(); stats.updateStatistic(region); logger.info("UPDATE STATISTICS finished successfully for scanner: " + innerScanner + ". Number of rows scanned: " + rowCount diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsInfoWriter.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsInfoWriter.java index f0963e0a970..c954d349a5a 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsInfoWriter.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsInfoWriter.java @@ -29,13 +29,12 @@ /* * Writer to help in writing guidePosts and creating guidePostInfo. This is used when we are collecting stats or reading stats for a table. - * updateGuidePosts() is required to be invoked before calling getGuidePostInfo() in order to get the updated guidePosts */ public class GuidePostsInfoWriter { private PrefixByteEncoder encoder; private byte[] lastRow; - private GuidePostsInfo guidePostInfo; + private ImmutableBytesWritable guidePosts=new ImmutableBytesWritable(ByteUtil.EMPTY_BYTE_ARRAY); private long byteCount = 0; private int guidePostsCount; @@ -50,7 +49,6 @@ public class GuidePostsInfoWriter { private int maxLength; private DataOutputStream output; private TrustedByteArrayOutputStream stream; - private ImmutableBytesWritable guidePosts=new ImmutableBytesWritable(); public final static GuidePostsInfo EMPTY_GUIDEPOST = new GuidePostsInfo(0, new ImmutableBytesWritable(ByteUtil.EMPTY_BYTE_ARRAY), 0, 0, 0); @@ -97,20 +95,6 @@ public boolean writeGuidePosts(byte[] row){ public boolean writeGuidePosts(byte[] row, long byteCount){ return writeGuidePosts(row, byteCount, 0); } - - /* - * This needs to be called once all stats are encoded and stream buffer needs to be copied for retrieval before - * close() - */ - public void updateGuidePosts() { - this.guidePosts.set(Bytes.copy(stream.getBuffer(), 0, stream.size())); - this.guidePostInfo = new GuidePostsInfo(this.byteCount, this.guidePosts, this.rowCount, this.maxLength, - this.guidePostsCount); - } - - public GuidePostsInfo getGuidePostInfo() { - return this.guidePostInfo; - } public void close() { CodecUtils.close(stream); @@ -119,4 +103,22 @@ public void close() { public void incrementRowCount() { this.rowCount++; } + + public long getByteCount() { + return byteCount; + } + + public int getGuidePostsCount() { + return guidePostsCount; + } + + public long getRowCount() { + return rowCount; + } + + public ImmutableBytesWritable getGuidePosts() { + this.guidePosts.set(stream.getBuffer(), 0, stream.size()); + return guidePosts; + } + } diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollector.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollector.java index 0c619640541..18f7f8a817d 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollector.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollector.java @@ -232,17 +232,8 @@ public void clear() { public GuidePostsInfo getGuidePosts(ImmutableBytesPtr fam) { Pair pair = guidePostsInfoWriterMap.get(fam); - if (pair != null) { return pair.getSecond().getGuidePostInfo(); } + if (pair != null) { return StatisticsUtil.getGuidePostsInfoFromWriter(pair.getSecond()); } return null; } - public void updateGuidePosts() { - for (ImmutableBytesPtr cf : guidePostsInfoWriterMap.keySet()) { - GuidePostsInfoWriter guidePostWriter = guidePostsInfoWriterMap.get(cf).getSecond(); - if (guidePostWriter != null) { - guidePostWriter.updateGuidePosts(); - } - } - } - } diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsScanner.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsScanner.java index ffce33674b3..13e9491e029 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsScanner.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsScanner.java @@ -93,7 +93,6 @@ protected void updateStat(final List results) { @Override public void close() throws IOException { boolean async = config.getBoolean(COMMIT_STATS_ASYNC, DEFAULT_COMMIT_STATS_ASYNC); - tracker.updateGuidePosts(); StatisticsCollectionRunTracker collectionTracker = StatisticsCollectionRunTracker.getInstance(config); StatisticsScannerCallable callable = new StatisticsScannerCallable(); if (!async) { diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsUtil.java index a1c63d71ce3..22542c37b9f 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsUtil.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsUtil.java @@ -180,12 +180,8 @@ public static PTableStats readStatistics(HTableInterface statsHTable, byte[] tab guidePostsInfoWriter.writeGuidePosts(newGPStartKey, byteCount, rowCount); } } - for (byte[] cf : guidePostsInfoWriterPerCf.keySet()) { - GuidePostsInfoWriter guidePostsInfoWriter = guidePostsInfoWriterPerCf.get(cf); - if (guidePostsInfoWriter != null) { - guidePostsInfoWriter.updateGuidePosts(); - } - } + if (!guidePostsInfoWriterPerCf.isEmpty()) { return new PTableStatsImpl( + getGuidePostsPerCf(guidePostsInfoWriterPerCf), timeStamp); } } finally { if (scanner != null) { scanner.close(); @@ -197,7 +193,6 @@ public static PTableStats readStatistics(HTableInterface statsHTable, byte[] tab } } } - if (!guidePostsInfoWriterPerCf.isEmpty()) { return new PTableStatsImpl(getGuidePostsPerCf(guidePostsInfoWriterPerCf), timeStamp); } return PTableStats.EMPTY_STATS; } @@ -205,7 +200,7 @@ private static SortedMap getGuidePostsPerCf( TreeMap guidePostsWriterPerCf) { TreeMap guidePostsPerCf = new TreeMap(Bytes.BYTES_COMPARATOR); for (byte[] key : guidePostsWriterPerCf.keySet()) { - guidePostsPerCf.put(key, guidePostsWriterPerCf.get(key).getGuidePostInfo()); + guidePostsPerCf.put(key, StatisticsUtil.getGuidePostsInfoFromWriter(guidePostsWriterPerCf.get(key))); } return guidePostsPerCf; } @@ -231,4 +226,9 @@ public static byte[] getGuidePostsInfoFromRowKey(byte[] tableNameBytes, byte[] f ptr.set(row, gpOffset, row.length - gpOffset); return ByteUtil.copyKeyBytesIfNecessary(ptr); } + + public static GuidePostsInfo getGuidePostsInfoFromWriter(GuidePostsInfoWriter writer) { + return new GuidePostsInfo(writer.getByteCount(), writer.getGuidePosts(), writer.getRowCount(), + writer.getMaxLength(), writer.getGuidePostsCount()); + } } \ No newline at end of file From fb1f3d698c529b0b816e01aba83990b98b9deaa8 Mon Sep 17 00:00:00 2001 From: Josh Mahonin Date: Sat, 16 Jan 2016 13:46:10 -0500 Subject: [PATCH 10/47] PHOENIX-2599 PhoenixRecordReader does not handle StaleRegionBoundaryCacheException --- .../coprocessor/BaseScannerRegionObserver.java | 10 +++++++++- .../apache/phoenix/mapreduce/PhoenixRecordReader.java | 11 +++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/BaseScannerRegionObserver.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/BaseScannerRegionObserver.java index 034848b1cac..c95cd5df194 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/BaseScannerRegionObserver.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/BaseScannerRegionObserver.java @@ -94,6 +94,7 @@ abstract public class BaseScannerRegionObserver extends BaseRegionObserver { public static final String UPGRADE_DESC_ROW_KEY = "_UPGRADE_DESC_ROW_KEY"; public static final String SCAN_REGION_SERVER = "_SCAN_REGION_SERVER"; public static final String RUN_UPDATE_STATS_ASYNC_ATTRIB = "_RunUpdateStatsAsync"; + public static final String SKIP_REGION_BOUNDARY_CHECK = "_SKIP_REGION_BOUNDARY_CHECK"; /** * Attribute name used to pass custom annotations in Scans and Mutations (later). Custom annotations @@ -145,11 +146,18 @@ private static void throwIfScanOutOfRegion(Scan scan, Region region) throws DoNo abstract protected boolean isRegionObserverFor(Scan scan); abstract protected RegionScanner doPostScannerOpen(ObserverContext c, final Scan scan, final RegionScanner s) throws Throwable; + protected boolean skipRegionBoundaryCheck(Scan scan) { + byte[] skipCheckBytes = scan.getAttribute(SKIP_REGION_BOUNDARY_CHECK); + return skipCheckBytes != null && Bytes.toBoolean(skipCheckBytes); + } + @Override public RegionScanner preScannerOpen(final ObserverContext c, final Scan scan, final RegionScanner s) throws IOException { if (isRegionObserverFor(scan)) { - throwIfScanOutOfRegion(scan, c.getEnvironment().getRegion()); + if (! skipRegionBoundaryCheck(scan)) { + throwIfScanOutOfRegion(scan, c.getEnvironment().getRegion()); + } // Muck with the start/stop row of the scan and set as reversed at the // last possible moment. You need to swap the start/stop and make the // start exclusive and the stop inclusive. diff --git a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/PhoenixRecordReader.java b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/PhoenixRecordReader.java index 2df76fcb261..51ac79550cd 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/PhoenixRecordReader.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/PhoenixRecordReader.java @@ -27,6 +27,7 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.io.NullWritable; import org.apache.hadoop.mapreduce.InputSplit; import org.apache.hadoop.mapreduce.RecordReader; @@ -35,6 +36,7 @@ import org.apache.hadoop.util.ReflectionUtils; import org.apache.phoenix.compile.QueryPlan; import org.apache.phoenix.compile.StatementContext; +import org.apache.phoenix.coprocessor.BaseScannerRegionObserver; import org.apache.phoenix.iterate.ConcatResultIterator; import org.apache.phoenix.iterate.LookAheadResultIterator; import org.apache.phoenix.iterate.PeekingResultIterator; @@ -48,6 +50,7 @@ import com.google.common.base.Preconditions; import com.google.common.base.Throwables; import com.google.common.collect.Lists; +import org.apache.phoenix.query.ConnectionQueryServices; /** * {@link RecordReader} implementation that iterates over the the records. @@ -107,8 +110,16 @@ public void initialize(InputSplit split, TaskAttemptContext context) throws IOEx StatementContext ctx = queryPlan.getContext(); ReadMetricQueue readMetrics = ctx.getReadMetricsQueue(); String tableName = queryPlan.getTableRef().getTable().getPhysicalName().getString(); + + // Clear the table region boundary cache to make sure long running jobs stay up to date + byte[] tableNameBytes = queryPlan.getTableRef().getTable().getPhysicalName().getBytes(); + ConnectionQueryServices services = queryPlan.getContext().getConnection().getQueryServices(); + services.clearTableRegionCache(tableNameBytes); + long renewScannerLeaseThreshold = queryPlan.getContext().getConnection().getQueryServices().getRenewLeaseThresholdMilliSeconds(); for (Scan scan : scans) { + // For MR, skip the region boundary check exception if we encounter a split. ref: PHOENIX-2599 + scan.setAttribute(BaseScannerRegionObserver.SKIP_REGION_BOUNDARY_CHECK, Bytes.toBytes(true)); final TableResultIterator tableResultIterator = new TableResultIterator(queryPlan.getContext().getConnection().getMutationState(), queryPlan.getTableRef(), scan, readMetrics.allotMetric(SCAN_BYTES, tableName), renewScannerLeaseThreshold); PeekingResultIterator peekingResultIterator = LookAheadResultIterator.wrap(tableResultIterator); From 04c3fbd381144d99a87005685cb2f68f6d171f90 Mon Sep 17 00:00:00 2001 From: Ankit Singhal Date: Sun, 17 Jan 2016 02:42:08 +0530 Subject: [PATCH 11/47] updated writer --- .../apache/phoenix/schema/stats/GuidePostsInfoWriter.java | 5 +++++ .../apache/phoenix/schema/stats/StatisticsCollector.java | 2 +- .../org/apache/phoenix/schema/stats/StatisticsUtil.java | 7 +------ 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsInfoWriter.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsInfoWriter.java index c954d349a5a..a47521f1a9f 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsInfoWriter.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsInfoWriter.java @@ -120,5 +120,10 @@ public ImmutableBytesWritable getGuidePosts() { this.guidePosts.set(stream.getBuffer(), 0, stream.size()); return guidePosts; } + + public GuidePostsInfo createGuidePostInfo(){ + return new GuidePostsInfo(this.getByteCount(), this.getGuidePosts(), this.getRowCount(), + this.getMaxLength(), this.getGuidePostsCount()); + } } diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollector.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollector.java index 18f7f8a817d..30e13bd37c8 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollector.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollector.java @@ -232,7 +232,7 @@ public void clear() { public GuidePostsInfo getGuidePosts(ImmutableBytesPtr fam) { Pair pair = guidePostsInfoWriterMap.get(fam); - if (pair != null) { return StatisticsUtil.getGuidePostsInfoFromWriter(pair.getSecond()); } + if (pair != null) { return pair.getSecond().createGuidePostInfo(); } return null; } diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsUtil.java index 22542c37b9f..41a92c1a675 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsUtil.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsUtil.java @@ -200,7 +200,7 @@ private static SortedMap getGuidePostsPerCf( TreeMap guidePostsWriterPerCf) { TreeMap guidePostsPerCf = new TreeMap(Bytes.BYTES_COMPARATOR); for (byte[] key : guidePostsWriterPerCf.keySet()) { - guidePostsPerCf.put(key, StatisticsUtil.getGuidePostsInfoFromWriter(guidePostsWriterPerCf.get(key))); + guidePostsPerCf.put(key, guidePostsWriterPerCf.get(key).createGuidePostInfo()); } return guidePostsPerCf; } @@ -226,9 +226,4 @@ public static byte[] getGuidePostsInfoFromRowKey(byte[] tableNameBytes, byte[] f ptr.set(row, gpOffset, row.length - gpOffset); return ByteUtil.copyKeyBytesIfNecessary(ptr); } - - public static GuidePostsInfo getGuidePostsInfoFromWriter(GuidePostsInfoWriter writer) { - return new GuidePostsInfo(writer.getByteCount(), writer.getGuidePosts(), writer.getRowCount(), - writer.getMaxLength(), writer.getGuidePostsCount()); - } } \ No newline at end of file From 2bcaf8748c81e14f03d6db387a2beda9b29bb137 Mon Sep 17 00:00:00 2001 From: Ankit Singhal Date: Sun, 17 Jan 2016 02:47:27 +0530 Subject: [PATCH 12/47] test case updated with new writer --- .../apache/phoenix/filter/SkipScanBigFilterTest.java | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanBigFilterTest.java b/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanBigFilterTest.java index b30f8f0505d..7c61ace7a0a 100644 --- a/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanBigFilterTest.java +++ b/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanBigFilterTest.java @@ -26,7 +26,6 @@ import java.util.Properties; import java.util.SortedMap; -import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import org.apache.hadoop.hbase.util.Bytes; import org.apache.phoenix.compile.QueryPlan; import org.apache.phoenix.end2end.Shadower; @@ -39,8 +38,8 @@ import org.apache.phoenix.schema.PTableImpl; import org.apache.phoenix.schema.PTableKey; import org.apache.phoenix.schema.stats.GuidePostsInfo; +import org.apache.phoenix.schema.stats.GuidePostsInfoWriter; import org.apache.phoenix.schema.stats.PTableStats; -import org.apache.phoenix.util.ByteUtil; import org.apache.phoenix.util.PropertiesUtil; import org.apache.phoenix.util.ReadOnlyProps; import org.junit.BeforeClass; @@ -648,12 +647,12 @@ private void testIntersect(byte[][] regionBoundaries, byte[][] guidePosts) throw stmt.execute(); final PTable table = conn.unwrap(PhoenixConnection.class).getTable(new PTableKey(null, "PERF.BIG_OLAP_DOC")); - GuidePostsInfo info = new GuidePostsInfo(0, new ImmutableBytesWritable(ByteUtil.EMPTY_BYTE_ARRAY), 0l, 0, 0); + GuidePostsInfoWriter gpWriter = new GuidePostsInfoWriter(); for (byte[] gp : guidePosts) { - info.encodeAndCollectGuidePost(gp, 1000); + gpWriter.writeGuidePosts(gp, 1000); } - info.updateGuidePosts(); - info.close(); + GuidePostsInfo info = gpWriter.createGuidePostInfo(); + gpWriter.close(); final SortedMap gpMap = Maps.newTreeMap(Bytes.BYTES_COMPARATOR); gpMap.put(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES, info); PTable tableWithStats = PTableImpl.makePTable(table, new PTableStats() { From 4b013b9c018e84ae11436c66515999400f77f511 Mon Sep 17 00:00:00 2001 From: James Taylor Date: Thu, 14 Jan 2016 18:15:52 -0800 Subject: [PATCH 13/47] PHOENIX-2597 Local Indexes are not used to query views --- .../apache/phoenix/end2end/BaseViewIT.java | 60 ++- .../org/apache/phoenix/end2end/ViewIT.java | 36 +- .../phoenix/compile/PostDDLCompiler.java | 2 +- .../apache/phoenix/schema/MetaDataClient.java | 389 +++++++++--------- 4 files changed, 269 insertions(+), 218 deletions(-) diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseViewIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseViewIT.java index c713df47246..7e7175fc0b1 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseViewIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseViewIT.java @@ -27,11 +27,20 @@ import java.sql.Connection; import java.sql.DriverManager; import java.sql.ResultSet; +import java.sql.Statement; import java.util.Arrays; import java.util.Collection; import java.util.List; import java.util.Map; +import org.apache.hadoop.hbase.client.HTableInterface; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.ResultScanner; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Pair; +import org.apache.phoenix.jdbc.PhoenixConnection; +import org.apache.phoenix.jdbc.PhoenixStatement; import org.apache.phoenix.query.KeyRange; import org.apache.phoenix.query.QueryServices; import org.apache.phoenix.util.QueryUtil; @@ -81,20 +90,40 @@ public static Collection data() { } protected void testUpdatableViewWithIndex(Integer saltBuckets, boolean localIndex) throws Exception { - testUpdatableView(saltBuckets); - testUpdatableViewIndex(saltBuckets, localIndex); + String viewName = testUpdatableView(saltBuckets); + Pair pair = testUpdatableViewIndex(saltBuckets, localIndex); + Scan scan = pair.getSecond(); + String tableName = pair.getFirst(); + // Confirm that dropping the view also deletes the rows in the index + if (saltBuckets == null) { + try (Connection conn = DriverManager.getConnection(getUrl())) { + HTableInterface htable = conn.unwrap(PhoenixConnection.class).getQueryServices().getTable(Bytes.toBytes(tableName)); + ResultScanner scanner = htable.getScanner(scan); + Result result = scanner.next(); + // Confirm index has rows + assertTrue(result != null && !result.isEmpty()); + + conn.createStatement().execute("DROP VIEW " + viewName); + + // Confirm index has no rows after view is dropped + scanner = htable.getScanner(scan); + result = scanner.next(); + assertTrue(result == null || result.isEmpty()); + } + } } - protected void testUpdatableView(Integer saltBuckets) throws Exception { + protected String testUpdatableView(Integer saltBuckets) throws Exception { Connection conn = DriverManager.getConnection(getUrl()); if (saltBuckets!=null) { if (tableDDLOptions.length()!=0) tableDDLOptions+=","; tableDDLOptions+=(" SALT_BUCKETS="+saltBuckets); } + String viewName = "V"; String ddl = "CREATE TABLE " + fullTableName + " (k1 INTEGER NOT NULL, k2 INTEGER NOT NULL, k3 DECIMAL, s VARCHAR CONSTRAINT pk PRIMARY KEY (k1, k2, k3))" + tableDDLOptions; conn.createStatement().execute(ddl); - ddl = "CREATE VIEW v AS SELECT * FROM " + fullTableName + " WHERE k1 = 1"; + ddl = "CREATE VIEW " + viewName + " AS SELECT * FROM " + fullTableName + " WHERE k1 = 1"; conn.createStatement().execute(ddl); for (int i = 0; i < 10; i++) { conn.createStatement().execute("UPSERT INTO " + fullTableName + " VALUES(" + (i % 4) + "," + (i+100) + "," + (i > 5 ? 2 : 1) + ")"); @@ -135,13 +164,14 @@ protected void testUpdatableView(Integer saltBuckets) throws Exception { assertEquals(121, rs.getInt(2)); assertFalse(rs.next()); conn.close(); + return viewName; } - protected void testUpdatableViewIndex(Integer saltBuckets) throws Exception { - testUpdatableViewIndex(saltBuckets, false); + protected Pair testUpdatableViewIndex(Integer saltBuckets) throws Exception { + return testUpdatableViewIndex(saltBuckets, false); } - protected void testUpdatableViewIndex(Integer saltBuckets, boolean localIndex) throws Exception { + protected Pair testUpdatableViewIndex(Integer saltBuckets, boolean localIndex) throws Exception { ResultSet rs; Connection conn = DriverManager.getConnection(getUrl()); if (localIndex) { @@ -196,28 +226,32 @@ protected void testUpdatableViewIndex(Integer saltBuckets, boolean localIndex) t query = "SELECT k1, k2, s FROM v WHERE s = 'foo'"; - rs = conn.createStatement().executeQuery(query); + Statement statement = conn.createStatement(); + rs = statement.executeQuery(query); + Scan scan = statement.unwrap(PhoenixStatement.class).getQueryPlan().getContext().getScan(); assertTrue(rs.next()); assertEquals(1, rs.getInt(1)); assertEquals(120, rs.getInt(2)); assertEquals("foo", rs.getString(3)); assertFalse(rs.next()); + String htableName; rs = conn.createStatement().executeQuery("EXPLAIN " + query); if (localIndex) { - assertEquals("CLIENT PARALLEL "+ (saltBuckets == null ? 1 : saltBuckets) +"-WAY RANGE SCAN OVER _LOCAL_IDX_" + tableName +" [" + (Short.MIN_VALUE+1) + ",'foo']\n" + htableName = "_LOCAL_IDX_" + tableName; + assertEquals("CLIENT PARALLEL "+ (saltBuckets == null ? 1 : saltBuckets) +"-WAY RANGE SCAN OVER " + htableName +" [" + (Short.MIN_VALUE+1) + ",'foo']\n" + " SERVER FILTER BY FIRST KEY ONLY\n" + "CLIENT MERGE SORT",QueryUtil.getExplainPlan(rs)); } else { + htableName = "_IDX_" + tableName; assertEquals(saltBuckets == null - ? "CLIENT PARALLEL 1-WAY RANGE SCAN OVER _IDX_" + tableName +" [" + (Short.MIN_VALUE+1) + ",'foo']\n" + ? "CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + htableName +" [" + (Short.MIN_VALUE+1) + ",'foo']\n" + " SERVER FILTER BY FIRST KEY ONLY" - : "CLIENT PARALLEL " + saltBuckets + "-WAY RANGE SCAN OVER _IDX_T" + (transactional ? "_TXN" : "") + " [0," + (Short.MIN_VALUE+1) + ",'foo']\n" + : "CLIENT PARALLEL " + saltBuckets + "-WAY RANGE SCAN OVER " + htableName + " [0," + (Short.MIN_VALUE+1) + ",'foo']\n" + " SERVER FILTER BY FIRST KEY ONLY\n" + "CLIENT MERGE SORT", QueryUtil.getExplainPlan(rs)); } conn.close(); + return new Pair<>(htableName,scan); } - - } diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ViewIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ViewIT.java index 55f26cd97c4..146fb1f9357 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ViewIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ViewIT.java @@ -450,13 +450,26 @@ private void validateViewDoesNotExist(Connection conn, String viewName) throws S } } - @Test - public void testViewUsesTableIndex() throws Exception { + @Test + public void testViewUsesTableGlobalIndex() throws Exception { + testViewUsesTableIndex(false); + } + + @Test + public void testViewUsesTableLocalIndex() throws Exception { + testViewUsesTableIndex(true); + } + + + private void testViewUsesTableIndex(boolean localIndex) throws Exception { ResultSet rs; + // Use unique name for table with local index as otherwise we run into issues + // when we attempt to drop the table (with the drop metadata option set to false + String fullTableName = this.fullTableName + (localIndex ? "_WITH_LI" : "_WITHOUT_LI"); Connection conn = DriverManager.getConnection(getUrl()); String ddl = "CREATE TABLE " + fullTableName + " (k1 INTEGER NOT NULL, k2 INTEGER NOT NULL, k3 DECIMAL, s1 VARCHAR, s2 VARCHAR CONSTRAINT pk PRIMARY KEY (k1, k2, k3))" + tableDDLOptions; conn.createStatement().execute(ddl); - conn.createStatement().execute("CREATE INDEX i1 ON " + fullTableName + "(k3, k2) INCLUDE(s1, s2)"); + conn.createStatement().execute("CREATE " + (localIndex ? "LOCAL " : "") + " INDEX i1 ON " + fullTableName + "(k3, k2) INCLUDE(s1, s2)"); conn.createStatement().execute("CREATE INDEX i2 ON " + fullTableName + "(k3, k2, s2)"); ddl = "CREATE VIEW v AS SELECT * FROM " + fullTableName + " WHERE s1 = 'foo'"; @@ -481,11 +494,18 @@ public void testViewUsesTableIndex() throws Exception { assertFalse(rs.next()); rs = conn.createStatement().executeQuery("EXPLAIN " + query); String queryPlan = QueryUtil.getExplainPlan(rs); - assertEquals( - "CLIENT PARALLEL 1-WAY SKIP SCAN ON 4 KEYS OVER I1 [1,100] - [2,109]\n" + - " SERVER FILTER BY (\"S2\" = 'bas' AND \"S1\" = 'foo')", queryPlan); - } - + // Assert that in either case (local & global) that index from physical table used for query on view. + if (localIndex) { + assertEquals("CLIENT PARALLEL 1-WAY SKIP SCAN ON 4 KEYS OVER _LOCAL_IDX_" + fullTableName + " [-32768,1,100] - [-32768,2,109]\n" + + " SERVER FILTER BY (\"S2\" = 'bas' AND \"S1\" = 'foo')\n" + + "CLIENT MERGE SORT", queryPlan); + } else { + assertEquals( + "CLIENT PARALLEL 1-WAY SKIP SCAN ON 4 KEYS OVER I1 [1,100] - [2,109]\n" + + " SERVER FILTER BY (\"S2\" = 'bas' AND \"S1\" = 'foo')", queryPlan); + } + } + @Test public void testCreateViewDefinesPKColumn() throws Exception { Connection conn = DriverManager.getConnection(getUrl()); diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/PostDDLCompiler.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/PostDDLCompiler.java index 192a031a21b..2659b3f8d8a 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/compile/PostDDLCompiler.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/PostDDLCompiler.java @@ -127,7 +127,7 @@ public boolean hasUDFs() { @Override public MutationState execute() throws SQLException { if (tableRefs.isEmpty()) { - return null; + return new MutationState(0, connection); } boolean wasAutoCommit = connection.getAutoCommit(); try { diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java index f1bb582fa72..ee8fba356af 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java @@ -113,8 +113,6 @@ import java.util.Properties; import java.util.Set; -import co.cask.tephra.TxConstants; - import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.HConstants; @@ -217,6 +215,8 @@ import com.google.common.collect.Sets; import com.google.common.primitives.Ints; +import co.cask.tephra.TxConstants; + public class MetaDataClient { private static final Logger logger = LoggerFactory.getLogger(MetaDataClient.class); @@ -441,7 +441,7 @@ private long getClientTimeStamp() { } private long getCurrentScn() { - Long scn = connection.getSCN(); + Long scn = connection.getSCN(); long currentScn = scn == null ? HConstants.LATEST_TIMESTAMP : scn; return currentScn; } @@ -465,20 +465,20 @@ private MetaDataMutationResult updateCache(PName tenantId, String schemaName, St } boolean defaultTransactional = connection.getQueryServices().getProps().getBoolean( - QueryServices.DEFAULT_TABLE_ISTRANSACTIONAL_ATTRIB, - QueryServicesOptions.DEFAULT_TRANSACTIONAL); + QueryServices.DEFAULT_TABLE_ISTRANSACTIONAL_ATTRIB, + QueryServicesOptions.DEFAULT_TRANSACTIONAL); // start a txn if all table are transactional by default or if we found the table in the cache and it is transactional // TODO if system tables become transactional remove the check boolean isTransactional = defaultTransactional || (table!=null && table.isTransactional()); if (!systemTable && isTransactional && !connection.getMutationState().isTransactionStarted()) { - connection.getMutationState().startTransaction(); + connection.getMutationState().startTransaction(); } resolvedTimestamp = resolvedTimestamp==null ? TransactionUtil.getResolvedTimestamp(connection, isTransactional, HConstants.LATEST_TIMESTAMP) : resolvedTimestamp; // Do not make rpc to getTable if // 1. table is a system table // 2. table was already resolved as of that timestamp - if (table != null && !alwaysHitServer - && (systemTable || resolvedTimestamp == tableResolvedTimestamp)) { + if (table != null && !alwaysHitServer + && (systemTable || resolvedTimestamp == tableResolvedTimestamp)) { return new MetaDataMutationResult(MutationCode.TABLE_ALREADY_EXISTS, QueryConstants.UNSET_TIMESTAMP, table); } @@ -490,11 +490,11 @@ private MetaDataMutationResult updateCache(PName tenantId, String schemaName, St final byte[] schemaBytes = PVarchar.INSTANCE.toBytes(schemaName); final byte[] tableBytes = PVarchar.INSTANCE.toBytes(tableName); ConnectionQueryServices queryServices = connection.getQueryServices(); - result = queryServices.getTable(tenantId, schemaBytes, tableBytes, tableTimestamp, resolvedTimestamp); - // if the table was assumed to be transactional, but is actually not transactional then re-resolve as of the right timestamp (and vice versa) - if (table==null && result.getTable()!=null && result.getTable().isTransactional()!=isTransactional) { - result = queryServices.getTable(tenantId, schemaBytes, tableBytes, tableTimestamp, TransactionUtil.getResolvedTimestamp(connection, result.getTable().isTransactional(), HConstants.LATEST_TIMESTAMP)); - } + result = queryServices.getTable(tenantId, schemaBytes, tableBytes, tableTimestamp, resolvedTimestamp); + // if the table was assumed to be transactional, but is actually not transactional then re-resolve as of the right timestamp (and vice versa) + if (table==null && result.getTable()!=null && result.getTable().isTransactional()!=isTransactional) { + result = queryServices.getTable(tenantId, schemaBytes, tableBytes, tableTimestamp, TransactionUtil.getResolvedTimestamp(connection, result.getTable().isTransactional(), HConstants.LATEST_TIMESTAMP)); + } if (SYSTEM_CATALOG_SCHEMA.equals(schemaName)) { return result; @@ -520,18 +520,18 @@ private MetaDataMutationResult updateCache(PName tenantId, String schemaName, St if (table != null) { // Ensures that table in result is set to table found in our cache. if (code == MutationCode.TABLE_ALREADY_EXISTS) { - result.setTable(table); - // Although this table is up-to-date, the parent table may not be. - // In this case, we update the parent table which may in turn pull - // in indexes to add to this table. - long resolvedTime = TransactionUtil.getResolvedTime(connection, result); - if (addIndexesFromPhysicalTable(result, resolvedTimestamp)) { - connection.addTable(result.getTable(), resolvedTime); - } - else { - // if we aren't adding the table, we still need to update the resolved time of the table - connection.updateResolvedTimestamp(table, resolvedTime); - } + result.setTable(table); + // Although this table is up-to-date, the parent table may not be. + // In this case, we update the parent table which may in turn pull + // in indexes to add to this table. + long resolvedTime = TransactionUtil.getResolvedTime(connection, result); + if (addIndexesFromPhysicalTable(result, resolvedTimestamp)) { + connection.addTable(result.getTable(), resolvedTime); + } + else { + // if we aren't adding the table, we still need to update the resolved time of the table + connection.updateResolvedTimestamp(table, resolvedTime); + } return result; } // If table was not found at the current time stamp and we have one cached, remove it. @@ -628,12 +628,12 @@ private MetaDataMutationResult updateCache(PName tenantId, List function * @throws SQLException if the physical table cannot be found */ private boolean addIndexesFromPhysicalTable(MetaDataMutationResult result, Long resolvedTimestamp) throws SQLException { - PTable table = result.getTable(); + PTable view = result.getTable(); // If not a view or if a view directly over an HBase table, there's nothing to do - if (table.getType() != PTableType.VIEW || table.getViewType() == ViewType.MAPPED) { + if (view.getType() != PTableType.VIEW || view.getViewType() == ViewType.MAPPED) { return false; } - String physicalName = table.getPhysicalName().getString(); + String physicalName = view.getPhysicalName().getString(); String schemaName = SchemaUtil.getSchemaNameFromFullName(physicalName); String tableName = SchemaUtil.getTableNameFromFullName(physicalName); MetaDataMutationResult parentResult = updateCache(null, schemaName, tableName, false, resolvedTimestamp); @@ -644,84 +644,72 @@ private boolean addIndexesFromPhysicalTable(MetaDataMutationResult result, Long if (!result.wasUpdated() && !parentResult.wasUpdated()) { return false; } - List indexes = physicalTable.getIndexes(); - if (indexes.isEmpty()) { + List physicalTableIndexes = physicalTable.getIndexes(); + if (physicalTableIndexes.isEmpty()) { return false; } // Filter out indexes if column doesn't exist in view - List allIndexes = Lists.newArrayListWithExpectedSize(indexes.size() + table.getIndexes().size()); + List indexesToAdd = Lists.newArrayListWithExpectedSize(physicalTableIndexes.size() + view.getIndexes().size()); if (result.wasUpdated()) { // Table from server never contains inherited indexes - allIndexes.addAll(table.getIndexes()); + indexesToAdd.addAll(view.getIndexes()); } else { // Only add original ones, as inherited ones may have changed - for (PTable index : indexes) { - if (index.getViewIndexId() != null) { - allIndexes.add(index); + for (PTable index : view.getIndexes()) { + // Original indexes will not have a view statement while inherited ones will + if (index.getViewStatement() == null) { + indexesToAdd.add(index); } } } - for (PTable index : indexes) { - if (index.getViewIndexId() == null) { - boolean containsAllReqdCols = true; - // Ensure that all columns required to create index - // exist in the view too (since view columns may be removed) - IndexMaintainer indexMaintainer = index.getIndexMaintainer(physicalTable, connection); - // check that the columns required for the index pk (not including the pk columns of the data table) - // are present in the view - Set indexColRefs = indexMaintainer.getIndexedColumns(); - for (ColumnReference colRef : indexColRefs) { - try { - byte[] cf= colRef.getFamily(); - byte[] cq= colRef.getQualifier(); - if (cf!=null) { - table.getColumnFamily(cf).getColumn(cq); - } - else { - table.getColumn( Bytes.toString(cq)); - } - } catch (ColumnNotFoundException e) { // Ignore this index and continue with others - containsAllReqdCols = false; - break; + for (PTable index : physicalTableIndexes) { + boolean containsAllReqdCols = true; + // Ensure that all columns required to create index exist in the view too, + // since view columns may be removed. + IndexMaintainer indexMaintainer = index.getIndexMaintainer(physicalTable, connection); + // Check that the columns required for the index pk are present in the view + Set indexColRefs = indexMaintainer.getIndexedColumns(); + for (ColumnReference colRef : indexColRefs) { + try { + byte[] cf= colRef.getFamily(); + byte[] cq= colRef.getQualifier(); + if (cf!=null) { + view.getColumnFamily(cf).getColumn(cq); + } + else { + view.getColumn( Bytes.toString(cq)); } + } catch (ColumnNotFoundException e) { // Ignore this index and continue with others + containsAllReqdCols = false; + break; } - // check that pk columns of the data table (which are also present in the index pk) are present in the view - List pkColumns = physicalTable.getPKColumns(); - for (int i = physicalTable.getBucketNum() == null ? 0 : 1; i < pkColumns.size(); i++) { + } + // Ensure that constant columns (i.e. columns matched in the view WHERE clause) + // all exist in the index on the physical table. + for (PColumn col : view.getColumns()) { + if (col.getViewConstant() != null) { try { - PColumn pkColumn = pkColumns.get(i); - table.getColumn(pkColumn.getName().getString()); + // It'd be possible to use a local index that doesn't have all view constants, + // but the WHERE clause for the view statement (which is added to the index below) + // would fail to compile. + String indexColumnName = IndexUtil.getIndexColumnName(col); + index.getColumn(indexColumnName); } catch (ColumnNotFoundException e) { // Ignore this index and continue with others containsAllReqdCols = false; break; } } - // Ensure that constant columns (i.e. columns matched in the view WHERE clause) - // all exist in the index on the physical table. - for (PColumn col : table.getColumns()) { - if (col.getViewConstant() != null) { - try { - // TODO: it'd be possible to use a local index that doesn't have all view constants - String indexColumnName = IndexUtil.getIndexColumnName(col); - index.getColumn(indexColumnName); - } catch (ColumnNotFoundException e) { // Ignore this index and continue with others - containsAllReqdCols = false; - break; - } - } - } - if (containsAllReqdCols) { - // Tack on view statement to index to get proper filtering for view - String viewStatement = IndexUtil.rewriteViewStatement(connection, index, physicalTable, table.getViewStatement()); - index = PTableImpl.makePTable(index, viewStatement); - allIndexes.add(index); - } + } + if (containsAllReqdCols) { + // Tack on view statement to index to get proper filtering for view + String viewStatement = IndexUtil.rewriteViewStatement(connection, index, physicalTable, view.getViewStatement()); + index = PTableImpl.makePTable(index, viewStatement); + indexesToAdd.add(index); } } - PTable allIndexesTable = PTableImpl.makePTable(table, table.getTimeStamp(), allIndexes); + PTable allIndexesTable = PTableImpl.makePTable(view, view.getTimeStamp(), indexesToAdd); result.setTable(allIndexesTable); return true; } - private void addColumnMutation(String schemaName, String tableName, PColumn column, PreparedStatement colUpsert, String parentTableName, String pkName, Short keySeq, boolean isSalted) throws SQLException { colUpsert.setString(1, connection.getTenantId() == null ? null : connection.getTenantId().getString()); colUpsert.setString(2, schemaName); @@ -1224,14 +1212,14 @@ public MutationState createIndex(CreateIndexStatement statement, byte[][] splits List pkColumns = dataTable.getPKColumns(); Set unusedPkColumns; if (dataTable.getBucketNum() != null) { // Ignore SALT column - unusedPkColumns = Sets.newLinkedHashSetWithExpectedSize(pkColumns.size()-1); - posOffset++; + unusedPkColumns = Sets.newLinkedHashSetWithExpectedSize(pkColumns.size()-1); + posOffset++; } else { - unusedPkColumns = Sets.newLinkedHashSetWithExpectedSize(pkColumns.size()); + unusedPkColumns = Sets.newLinkedHashSetWithExpectedSize(pkColumns.size()); } for (int i = posOffset; i < pkColumns.size(); i++) { PColumn column = pkColumns.get(i); - unusedPkColumns.add(new RowKeyColumnExpression(column, new RowKeyValueAccessor(pkColumns, i), "\""+column.getName().getString()+"\"")); + unusedPkColumns.add(new RowKeyColumnExpression(column, new RowKeyValueAccessor(pkColumns, i), "\""+column.getName().getString()+"\"")); } List allPkColumns = Lists.newArrayListWithExpectedSize(unusedPkColumns.size()); List columnDefs = Lists.newArrayListWithExpectedSize(includedColumns.size() + indexParseNodeAndSortOrderList.size()); @@ -1240,7 +1228,7 @@ public MutationState createIndex(CreateIndexStatement statement, byte[][] splits // Add tenant ID column as first column in index PColumn col = dataTable.getPKColumns().get(posOffset); RowKeyColumnExpression columnExpression = new RowKeyColumnExpression(col, new RowKeyValueAccessor(pkColumns, posOffset), col.getName().getString()); - unusedPkColumns.remove(columnExpression); + unusedPkColumns.remove(columnExpression); PDataType dataType = IndexUtil.getIndexColumnDataType(col); ColumnName colName = ColumnName.caseSensitiveColumnName(IndexUtil.getIndexColumnName(col)); allPkColumns.add(new ColumnDefInPkConstraint(colName, col.getSortOrder(), false)); @@ -1265,7 +1253,7 @@ public MutationState createIndex(CreateIndexStatement statement, byte[][] splits IndexExpressionCompiler expressionIndexCompiler = new IndexExpressionCompiler(context); Set indexedColumnNames = Sets.newHashSetWithExpectedSize(indexParseNodeAndSortOrderList.size()); for (Pair pair : indexParseNodeAndSortOrderList) { - ParseNode parseNode = pair.getFirst(); + ParseNode parseNode = pair.getFirst(); // normalize the parse node parseNode = StatementNormalizer.normalize(parseNode, resolver); // compile the parseNode to get an expression @@ -1291,22 +1279,22 @@ public MutationState createIndex(CreateIndexStatement statement, byte[][] splits ColumnName colName = null; ColumnRef colRef = expressionIndexCompiler.getColumnRef(); - boolean isRowTimestamp = false; + boolean isRowTimestamp = false; if (colRef!=null) { - // if this is a regular column - PColumn column = colRef.getColumn(); - String columnFamilyName = column.getFamilyName()!=null ? column.getFamilyName().getString() : null; - colName = ColumnName.caseSensitiveColumnName(IndexUtil.getIndexColumnName(columnFamilyName, column.getName().getString())); - isRowTimestamp = column.isRowTimestamp(); - } - else { - // if this is an expression - // TODO column names cannot have double quotes, remove this once this PHOENIX-1621 is fixed - String name = expressionStr.replaceAll("\"", "'"); + // if this is a regular column + PColumn column = colRef.getColumn(); + String columnFamilyName = column.getFamilyName()!=null ? column.getFamilyName().getString() : null; + colName = ColumnName.caseSensitiveColumnName(IndexUtil.getIndexColumnName(columnFamilyName, column.getName().getString())); + isRowTimestamp = column.isRowTimestamp(); + } + else { + // if this is an expression + // TODO column names cannot have double quotes, remove this once this PHOENIX-1621 is fixed + String name = expressionStr.replaceAll("\"", "'"); colName = ColumnName.caseSensitiveColumnName(IndexUtil.getIndexColumnName(null, name)); - } - indexedColumnNames.add(colName); - PDataType dataType = IndexUtil.getIndexColumnDataType(expression.isNullable(), expression.getDataType()); + } + indexedColumnNames.add(colName); + PDataType dataType = IndexUtil.getIndexColumnDataType(expression.isNullable(), expression.getDataType()); allPkColumns.add(new ColumnDefInPkConstraint(colName, pair.getSecond(), isRowTimestamp)); columnDefs.add(FACTORY.columnDef(colName, dataType.getSqlTypeName(), expression.isNullable(), expression.getMaxLength(), expression.getScale(), false, pair.getSecond(), expressionStr, isRowTimestamp)); } @@ -1593,7 +1581,7 @@ private PTable createTableInternal(CreateTableStatement statement, byte[][] spli // table instead of only a view? We don't have anywhere to put the link // from the table to the index, though. if (indexType == IndexType.LOCAL || (parent.getType() == PTableType.VIEW && parent.getViewType() != ViewType.MAPPED)) { - PName physicalName = parent.getPhysicalName(); + PName physicalName = parent.getPhysicalName(); saltBucketNum = parent.getBucketNum(); addSaltColumn = (saltBucketNum != null && indexType != IndexType.LOCAL); defaultFamilyName = parent.getDefaultFamilyName() == null ? null : parent.getDefaultFamilyName().getString(); @@ -1728,8 +1716,8 @@ private PTable createTableInternal(CreateTableStatement statement, byte[][] spli } } boolean transactionsEnabled = connection.getQueryServices().getProps().getBoolean( - QueryServices.TRANSACTIONS_ENABLED, - QueryServicesOptions.DEFAULT_TRANSACTIONS_ENABLED); + QueryServices.TRANSACTIONS_ENABLED, + QueryServicesOptions.DEFAULT_TRANSACTIONS_ENABLED); // can't create a transactional table if transactions are not enabled if (!transactionsEnabled && transactional) { throw new SQLExceptionInfo.Builder(SQLExceptionCode.CANNOT_CREATE_TXN_TABLE_IF_TXNS_DISABLED) @@ -1738,7 +1726,7 @@ private PTable createTableInternal(CreateTableStatement statement, byte[][] spli } // can't create a transactional table if it has a row timestamp column if (pkConstraint.getNumColumnsWithRowTimestamp()>0 && transactional) { - throw new SQLExceptionInfo.Builder(SQLExceptionCode.CANNOT_CREATE_TXN_TABLE_WITH_ROW_TIMESTAMP) + throw new SQLExceptionInfo.Builder(SQLExceptionCode.CANNOT_CREATE_TXN_TABLE_WITH_ROW_TIMESTAMP) .setSchemaName(schemaName).setTableName(tableName) .build().buildException(); } @@ -1762,16 +1750,16 @@ private PTable createTableInternal(CreateTableStatement statement, byte[][] spli // on the server, but this can mess up our secondary index code as the changes get // committed prior to the // maintenance code being able to see the prior state to update the rows correctly. - if (Boolean.FALSE.equals(storeNullsProp)) { - throw new SQLExceptionInfo.Builder(SQLExceptionCode.STORE_NULLS_MUST_BE_TRUE_FOR_TRANSACTIONAL) - .setSchemaName(schemaName).setTableName(tableName) - .build().buildException(); - } - // Force STORE_NULLS to true when transactional as Tephra cannot deal with column deletes - storeNulls = true; - tableProps.put(PhoenixDatabaseMetaData.STORE_NULLS, Boolean.TRUE); - - if (!sharedTable) { + if (Boolean.FALSE.equals(storeNullsProp)) { + throw new SQLExceptionInfo.Builder(SQLExceptionCode.STORE_NULLS_MUST_BE_TRUE_FOR_TRANSACTIONAL) + .setSchemaName(schemaName).setTableName(tableName) + .build().buildException(); + } + // Force STORE_NULLS to true when transactional as Tephra cannot deal with column deletes + storeNulls = true; + tableProps.put(PhoenixDatabaseMetaData.STORE_NULLS, Boolean.TRUE); + + if (!sharedTable) { Integer maxVersionsProp = (Integer) commonFamilyProps.get(HConstants.VERSIONS); if (maxVersionsProp == null) { if (parent != null) { @@ -1787,7 +1775,7 @@ private PTable createTableInternal(CreateTableStatement statement, byte[][] spli } commonFamilyProps.put(HConstants.VERSIONS, maxVersionsProp); } - } + } } timestamp = timestamp==null ? TransactionUtil.getTableTimestamp(connection, transactional) : timestamp; @@ -2016,8 +2004,8 @@ private PTable createTableInternal(CreateTableStatement statement, byte[][] spli } if (familyNames.isEmpty()) { - //if there are no family names, use the default column family name. This also takes care of the case when - //the table ddl has only PK cols present (which means familyNames is empty). + //if there are no family names, use the default column family name. This also takes care of the case when + //the table ddl has only PK cols present (which means familyNames is empty). byte[] cf = defaultFamilyName == null ? QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES : Bytes.toBytes(defaultFamilyName); familyPropList.add(new Pair>(cf, commonFamilyProps)); } @@ -2353,7 +2341,6 @@ private MutationState dropTable(String schemaName, String tableName, String pare MetaDataMutationResult result = connection.getQueryServices().dropTable(tableMetaData, tableType, cascade); MutationCode code = result.getMutationCode(); PTable table = result.getTable(); - boolean transactional = table!=null && table.isTransactional(); switch (code) { case TABLE_NOT_FOUND: if (!ifExists) { throw new TableNotFoundException(schemaName, tableName); } @@ -2365,54 +2352,64 @@ private MutationState dropTable(String schemaName, String tableName, String pare .setSchemaName(schemaName).setTableName(tableName).build().buildException(); default: - connection.removeTable(tenantId, SchemaUtil.getTableName(schemaName, tableName), parentTableName, result.getMutationTime()); + connection.removeTable(tenantId, SchemaUtil.getTableName(schemaName, tableName), parentTableName, result.getMutationTime()); - if (result.getTable() != null && tableType != PTableType.VIEW) { - connection.setAutoCommit(true); - boolean dropMetaData = result.getTable().getViewIndexId() == null && - connection.getQueryServices().getProps().getBoolean(DROP_METADATA_ATTRIB, DEFAULT_DROP_METADATA); + if (result.getTable() != null) { + boolean dropMetaData = false; long ts = (scn == null ? result.getMutationTime() : scn); - // Create empty table and schema - they're only used to get the name from - // PName name, PTableType type, long timeStamp, long sequenceNumber, List columns List tableRefs = Lists.newArrayListWithExpectedSize(2 + table.getIndexes().size()); - // All multi-tenant tables have a view index table, so no need to check in that case - if (tableType == PTableType.TABLE - && (table.isMultiTenant() || hasViewIndexTable || hasLocalIndexTable)) { - - MetaDataUtil.deleteViewIndexSequences(connection, table.getPhysicalName()); - if (hasViewIndexTable) { - String viewIndexSchemaName = null; - String viewIndexTableName = null; - if (schemaName != null) { - viewIndexSchemaName = MetaDataUtil.getViewIndexTableName(schemaName); - viewIndexTableName = tableName; - } else { - viewIndexTableName = MetaDataUtil.getViewIndexTableName(tableName); + connection.setAutoCommit(true); + if (tableType == PTableType.VIEW) { + for (PTable index : table.getIndexes()) { + // Don't drop index data from indexes on physical table + if (index.getViewStatement() == null) { + tableRefs.add(new TableRef(null, index, ts, false)); } - PTable viewIndexTable = new PTableImpl(null, viewIndexSchemaName, viewIndexTableName, ts, - table.getColumnFamilies()); - tableRefs.add(new TableRef(null, viewIndexTable, ts, false)); } - if (hasLocalIndexTable) { - String localIndexSchemaName = null; - String localIndexTableName = null; - if (schemaName != null) { - localIndexSchemaName = MetaDataUtil.getLocalIndexTableName(schemaName); - localIndexTableName = tableName; - } else { - localIndexTableName = MetaDataUtil.getLocalIndexTableName(tableName); + } else { + dropMetaData = result.getTable().getViewIndexId() == null && + connection.getQueryServices().getProps().getBoolean(DROP_METADATA_ATTRIB, DEFAULT_DROP_METADATA); + // Create empty table and schema - they're only used to get the name from + // PName name, PTableType type, long timeStamp, long sequenceNumber, List columns + // All multi-tenant tables have a view index table, so no need to check in that case + if (tableType == PTableType.TABLE + && (table.isMultiTenant() || hasViewIndexTable || hasLocalIndexTable)) { + + MetaDataUtil.deleteViewIndexSequences(connection, table.getPhysicalName()); + if (hasViewIndexTable) { + String viewIndexSchemaName = null; + String viewIndexTableName = null; + if (schemaName != null) { + viewIndexSchemaName = MetaDataUtil.getViewIndexTableName(schemaName); + viewIndexTableName = tableName; + } else { + viewIndexTableName = MetaDataUtil.getViewIndexTableName(tableName); + } + PTable viewIndexTable = new PTableImpl(null, viewIndexSchemaName, viewIndexTableName, ts, + table.getColumnFamilies()); + tableRefs.add(new TableRef(null, viewIndexTable, ts, false)); + } + if (hasLocalIndexTable) { + String localIndexSchemaName = null; + String localIndexTableName = null; + if (schemaName != null) { + localIndexSchemaName = MetaDataUtil.getLocalIndexTableName(schemaName); + localIndexTableName = tableName; + } else { + localIndexTableName = MetaDataUtil.getLocalIndexTableName(tableName); + } + PTable localIndexTable = new PTableImpl(null, localIndexSchemaName, localIndexTableName, + ts, Collections. emptyList()); + tableRefs.add(new TableRef(null, localIndexTable, ts, false)); } - PTable localIndexTable = new PTableImpl(null, localIndexSchemaName, localIndexTableName, - ts, Collections. emptyList()); - tableRefs.add(new TableRef(null, localIndexTable, ts, false)); } + tableRefs.add(new TableRef(null, table, ts, false)); + // TODO: Let the standard mutable secondary index maintenance handle this? + for (PTable index : table.getIndexes()) { + tableRefs.add(new TableRef(null, index, ts, false)); + } + deleteFromStatsTable(tableRefs, ts); } - tableRefs.add(new TableRef(null, table, ts, false)); - // TODO: Let the standard mutable secondary index maintenance handle this? - for (PTable index : table.getIndexes()) { - tableRefs.add(new TableRef(null, index, ts, false)); - } - deleteFromStatsTable(tableRefs, ts); if (!dropMetaData) { MutationPlan plan = new PostDDLCompiler(connection).compile(tableRefs, null, null, Collections. emptyList(), ts); @@ -2589,7 +2586,7 @@ public MutationState addColumn(AddColumnStatement statement) throws SQLException ListMultimap> stmtProperties = statement.getProps(); Map>> properties = new HashMap<>(stmtProperties.size()); TableRef tableRef = FromCompiler.getResolver(statement, connection).getTables().get(0); - PTable table = tableRef.getTable(); + PTable table = tableRef.getTable(); List columnDefs = statement.getColumnDefs(); if (columnDefs == null) { columnDefs = Collections.emptyList(); @@ -2686,15 +2683,15 @@ public MutationState addColumn(AddColumnStatement statement) throws SQLException } // cannot create a transactional table if transactions are disabled boolean transactionsEnabled = connection.getQueryServices().getProps().getBoolean( - QueryServices.TRANSACTIONS_ENABLED, - QueryServicesOptions.DEFAULT_TRANSACTIONS_ENABLED); + QueryServices.TRANSACTIONS_ENABLED, + QueryServicesOptions.DEFAULT_TRANSACTIONS_ENABLED); if (!transactionsEnabled) { - throw new SQLExceptionInfo.Builder(SQLExceptionCode.CANNOT_ALTER_TO_BE_TXN_IF_TXNS_DISABLED) + throw new SQLExceptionInfo.Builder(SQLExceptionCode.CANNOT_ALTER_TO_BE_TXN_IF_TXNS_DISABLED) .setSchemaName(schemaName).setTableName(tableName).build().buildException(); } // cannot make a table transactional if it has a row timestamp column if (SchemaUtil.hasRowTimestampColumn(table)) { - throw new SQLExceptionInfo.Builder(SQLExceptionCode.CANNOT_ALTER_TO_BE_TXN_WITH_ROW_TIMESTAMP) + throw new SQLExceptionInfo.Builder(SQLExceptionCode.CANNOT_ALTER_TO_BE_TXN_WITH_ROW_TIMESTAMP) .setSchemaName(schemaName).setTableName(tableName) .build().buildException(); } @@ -2748,20 +2745,20 @@ public MutationState addColumn(AddColumnStatement statement) throws SQLException // Add any new PK columns to end of index PK if (numPkColumnsAdded>0) { - // create PK column list that includes the newly created columns - List pkColumns = Lists.newArrayListWithExpectedSize(table.getPKColumns().size()+numPkColumnsAdded); - pkColumns.addAll(table.getPKColumns()); - for (int i=0; i pkColumns = Lists.newArrayListWithExpectedSize(table.getPKColumns().size()+numPkColumnsAdded); + pkColumns.addAll(table.getPKColumns()); + for (int i=0; i columnRefs = statement.getColumnRefs(); if(columnRefs == null) { columnRefs = Lists.newArrayListWithCapacity(0); @@ -3022,7 +3019,7 @@ public MutationState dropColumn(DropColumnStatement statement) throws SQLExcepti dropColumnMutations(table, tableColumnsToDrop, tableMetaData); for (PTable index : table.getIndexes()) { - IndexMaintainer indexMaintainer = index.getIndexMaintainer(table, connection); + IndexMaintainer indexMaintainer = index.getIndexMaintainer(table, connection); // get the columns required for the index pk Set indexColumns = indexMaintainer.getIndexedColumns(); // get the covered columns @@ -3030,11 +3027,11 @@ public MutationState dropColumn(DropColumnStatement statement) throws SQLExcepti List indexColumnsToDrop = Lists.newArrayListWithExpectedSize(columnRefs.size()); for(PColumn columnToDrop : tableColumnsToDrop) { ColumnReference columnToDropRef = new ColumnReference(columnToDrop.getFamilyName().getBytes(), columnToDrop.getName().getBytes()); - if (indexColumns.contains(columnToDropRef)) { + if (indexColumns.contains(columnToDropRef)) { indexesToDrop.add(new TableRef(index)); } else if (coveredColumns.contains(columnToDropRef)) { - String indexColumnName = IndexUtil.getIndexColumnName(columnToDrop); + String indexColumnName = IndexUtil.getIndexColumnName(columnToDrop); indexColumnsToDrop.add(index.getColumn(indexColumnName)); } } @@ -3186,7 +3183,7 @@ public MutationState alterIndex(AlterIndexStatement statement) throws SQLExcepti } } Long timeStamp = indexRef.getTable().isTransactional() ? indexRef.getTimeStamp() : null; - List tableMetadata = connection.getMutationState().toMutations(timeStamp).next().getSecond(); + List tableMetadata = connection.getMutationState().toMutations(timeStamp).next().getSecond(); connection.rollback(); MetaDataMutationResult result = connection.getQueryServices().updateIndexState(tableMetadata, dataTableName); @@ -3256,7 +3253,7 @@ public PTableStats getTableStats(PTable table) throws SQLException { */ boolean isSharedIndex = table.getViewIndexId() != null; if (isSharedIndex) { - // we are assuming the stats table is not transactional + // we are assuming the stats table is not transactional return connection.getQueryServices().getTableStats(table.getPhysicalName().getBytes(), getCurrentScn()); } boolean isView = table.getType() == PTableType.VIEW; @@ -3297,7 +3294,7 @@ private boolean isLastPKVariableLength(PTable table) { } private PTable getParentOfView(PTable view) throws SQLException { - //TODO just use view.getParentName().getString() after implementing https://issues.apache.org/jira/browse/PHOENIX-2114 + //TODO just use view.getParentName().getString() after implementing https://issues.apache.org/jira/browse/PHOENIX-2114 SelectStatement select = new SQLParser(view.getViewStatement()).parseQuery(); String parentName = SchemaUtil.normalizeFullTableName(select.getFrom().toString().trim()); return connection.getTable(new PTableKey(view.getTenantId(), parentName)); From 3e5fa3efceeee2da6b551f11c7bc7cfb86f06143 Mon Sep 17 00:00:00 2001 From: James Taylor Date: Sat, 16 Jan 2016 18:00:50 -0800 Subject: [PATCH 14/47] PHOENIX-2592 Ignore flaky test --- .../test/java/org/apache/phoenix/pherf/RuleGeneratorTest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/RuleGeneratorTest.java b/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/RuleGeneratorTest.java index c2eae864d7e..71b785b083d 100644 --- a/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/RuleGeneratorTest.java +++ b/phoenix-pherf/src/test/java/org/apache/phoenix/pherf/RuleGeneratorTest.java @@ -49,6 +49,7 @@ public class RuleGeneratorTest { private static final String matcherScenario = PherfConstants.SCENARIO_ROOT_PATTERN + ".xml"; + @Ignore("Flapping - see PHOENIX-2592") @Test public void testDateGenerator() throws Exception { XMLConfigParser parser = new XMLConfigParser(matcherScenario); From 028a7586c0151a1444df3089214fb522ca6ba8f9 Mon Sep 17 00:00:00 2001 From: Ankit Singhal Date: Sun, 17 Jan 2016 14:00:12 +0530 Subject: [PATCH 15/47] Converted GuidePostsInfoWritr to GuidePostsInfoBuilder --- .../{GuidePostsInfoWriter.java => GuidePostsInfoBuilder.java} | 0 .../apache/phoenix/util/{CodecUtils.java => PrefixByteCodec.java} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename phoenix-core/src/main/java/org/apache/phoenix/schema/stats/{GuidePostsInfoWriter.java => GuidePostsInfoBuilder.java} (100%) rename phoenix-core/src/main/java/org/apache/phoenix/util/{CodecUtils.java => PrefixByteCodec.java} (100%) diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsInfoWriter.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsInfoBuilder.java similarity index 100% rename from phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsInfoWriter.java rename to phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsInfoBuilder.java diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/CodecUtils.java b/phoenix-core/src/main/java/org/apache/phoenix/util/PrefixByteCodec.java similarity index 100% rename from phoenix-core/src/main/java/org/apache/phoenix/util/CodecUtils.java rename to phoenix-core/src/main/java/org/apache/phoenix/util/PrefixByteCodec.java From 3d952d3b6b146eab097e12d21fd2caad3c073ce9 Mon Sep 17 00:00:00 2001 From: Ankit Singhal Date: Sun, 17 Jan 2016 14:00:25 +0530 Subject: [PATCH 16/47] Converted GuidePostsInfoWritr to GuidePostsInfoBuilder --- .../phoenix/iterate/BaseResultIterators.java | 8 ++-- .../schema/stats/GuidePostsInfoBuilder.java | 48 +++++++------------ .../phoenix/schema/stats/PTableStatsImpl.java | 8 ++-- .../schema/stats/StatisticsCollector.java | 30 +++++------- .../phoenix/schema/stats/StatisticsUtil.java | 18 +++---- .../schema/stats/StatisticsWriter.java | 4 +- .../apache/phoenix/util/PrefixByteCodec.java | 2 +- .../phoenix/filter/SkipScanBigFilterTest.java | 9 ++-- .../util/PrefixByteEncoderDecoderTest.java | 10 ++-- 9 files changed, 54 insertions(+), 83 deletions(-) diff --git a/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java b/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java index d1300b54429..b3235e277da 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java @@ -81,7 +81,7 @@ import org.apache.phoenix.schema.stats.GuidePostsInfo; import org.apache.phoenix.schema.stats.PTableStats; import org.apache.phoenix.util.ByteUtil; -import org.apache.phoenix.util.CodecUtils; +import org.apache.phoenix.util.PrefixByteCodec; import org.apache.phoenix.util.LogUtil; import org.apache.phoenix.util.PrefixByteDecoder; import org.apache.phoenix.util.SQLCloseables; @@ -506,7 +506,7 @@ private List> getParallelScans(byte[] startKey, byte[] stopKey) throw input = new DataInputStream(stream); decoder = new PrefixByteDecoder(gps.getMaxLength()); try { - while (currentKey.compareTo(currentGuidePost = CodecUtils.decode(decoder, input)) >= 0 + while (currentKey.compareTo(currentGuidePost = PrefixByteCodec.decode(decoder, input)) >= 0 && currentKey.getLength() != 0) { guideIndex++; } @@ -535,7 +535,7 @@ private List> getParallelScans(byte[] startKey, byte[] stopKey) throw false); scans = addNewScan(parallelScans, scans, newScan, currentGuidePostBytes, false, regionLocation); currentKeyBytes = currentGuidePost.copyBytes(); - currentGuidePost = CodecUtils.decode(decoder, input); + currentGuidePost = PrefixByteCodec.decode(decoder, input); currentGuidePostBytes = currentGuidePost.copyBytes(); guideIndex++; } @@ -555,7 +555,7 @@ private List> getParallelScans(byte[] startKey, byte[] stopKey) throw if (!scans.isEmpty()) { // Add any remaining scans parallelScans.add(scans); } - CodecUtils.close(stream); + PrefixByteCodec.close(stream); return parallelScans; } diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsInfoBuilder.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsInfoBuilder.java index a47521f1a9f..11a1faaab7c 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsInfoBuilder.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsInfoBuilder.java @@ -23,7 +23,7 @@ import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import org.apache.hadoop.hbase.util.Bytes; import org.apache.phoenix.util.ByteUtil; -import org.apache.phoenix.util.CodecUtils; +import org.apache.phoenix.util.PrefixByteCodec; import org.apache.phoenix.util.PrefixByteEncoder; import org.apache.phoenix.util.TrustedByteArrayOutputStream; @@ -31,7 +31,7 @@ * Writer to help in writing guidePosts and creating guidePostInfo. This is used when we are collecting stats or reading stats for a table. */ -public class GuidePostsInfoWriter { +public class GuidePostsInfoBuilder { private PrefixByteEncoder encoder; private byte[] lastRow; private ImmutableBytesWritable guidePosts=new ImmutableBytesWritable(ByteUtil.EMPTY_BYTE_ARRAY); @@ -56,7 +56,7 @@ public class GuidePostsInfoWriter { public int getMaxLength() { return maxLength; } - public GuidePostsInfoWriter(){ + public GuidePostsInfoBuilder(){ this.stream = new TrustedByteArrayOutputStream(1); this.output = new DataOutputStream(stream); this.encoder=new PrefixByteEncoder(); @@ -71,7 +71,7 @@ public GuidePostsInfoWriter(){ * @return * @throws IOException */ - public boolean writeGuidePosts( byte[] row, long byteCount, long rowCount) { + public boolean addGuidePosts( byte[] row, long byteCount, long rowCount) { if (row.length != 0 && Bytes.compareTo(lastRow, row) < 0) { try { encoder.encode(output, row, 0, row.length); @@ -88,42 +88,26 @@ public boolean writeGuidePosts( byte[] row, long byteCount, long rowCount) { return false; } - public boolean writeGuidePosts(byte[] row){ - return writeGuidePosts(row, 0, 0); + public boolean addGuidePosts(byte[] row){ + return addGuidePosts(row, 0, 0); } - public boolean writeGuidePosts(byte[] row, long byteCount){ - return writeGuidePosts(row, byteCount, 0); + public boolean addGuidePosts(byte[] row, long byteCount){ + return addGuidePosts(row, byteCount, 0); } - public void close() { - CodecUtils.close(stream); + private void close() { + PrefixByteCodec.close(stream); } - public void incrementRowCount() { - this.rowCount++; - } - - public long getByteCount() { - return byteCount; - } - - public int getGuidePostsCount() { - return guidePostsCount; - } - - public long getRowCount() { - return rowCount; - } - - public ImmutableBytesWritable getGuidePosts() { + public GuidePostsInfo build() { this.guidePosts.set(stream.getBuffer(), 0, stream.size()); - return guidePosts; + GuidePostsInfo guidePostsInfo = new GuidePostsInfo(this.byteCount, this.guidePosts, this.rowCount, this.maxLength, this.guidePostsCount); + this.close(); + return guidePostsInfo; } - - public GuidePostsInfo createGuidePostInfo(){ - return new GuidePostsInfo(this.getByteCount(), this.getGuidePosts(), this.getRowCount(), - this.getMaxLength(), this.getGuidePostsCount()); + public void incrementRowCount() { + this.rowCount++; } } diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/PTableStatsImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/PTableStatsImpl.java index 81c73feabd0..dacc213e093 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/PTableStatsImpl.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/PTableStatsImpl.java @@ -28,7 +28,7 @@ import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import org.apache.hadoop.hbase.util.Bytes; import org.apache.phoenix.coprocessor.MetaDataProtocol; -import org.apache.phoenix.util.CodecUtils; +import org.apache.phoenix.util.PrefixByteCodec; import org.apache.phoenix.util.PrefixByteDecoder; import org.apache.phoenix.util.SizedUtil; @@ -82,20 +82,20 @@ public String toString() { PrefixByteDecoder decoder = new PrefixByteDecoder(entry.getValue().getMaxLength()); try { while (true) { - ImmutableBytesWritable ptr = CodecUtils.decode(decoder, input); + ImmutableBytesWritable ptr = PrefixByteCodec.decode(decoder, input); buf.append(Bytes.toStringBinary(ptr.get())); buf.append(","); } } catch (EOFException e) { // Ignore as this signifies we're done } finally { - CodecUtils.close(stream); + PrefixByteCodec.close(stream); } buf.setLength(buf.length() - 1); } buf.append(")"); } finally { - CodecUtils.close(stream); + PrefixByteCodec.close(stream); } } buf.append("]"); diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollector.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollector.java index 30e13bd37c8..3462f2235ca 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollector.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollector.java @@ -60,9 +60,9 @@ public class StatisticsCollector { private long guidepostDepth; private long maxTimeStamp = MetaDataProtocol.MIN_TABLE_TIMESTAMP; - private Map> guidePostsInfoWriterMap = Maps.newHashMap(); + private Map> guidePostsInfoWriterMap = Maps.newHashMap(); protected StatisticsWriter statsTable; - private Pair cachedGps = null; + private Pair cachedGps = null; public StatisticsCollector(RegionCoprocessorEnvironment env, String tableName, long clientTimeStamp) throws IOException { @@ -98,7 +98,7 @@ public StatisticsCollector(RegionCoprocessorEnvironment env, String tableName, l // in a compaction we know the one family ahead of time if (family != null) { ImmutableBytesPtr cfKey = new ImmutableBytesPtr(family); - cachedGps = new Pair(0l, new GuidePostsInfoWriter()); + cachedGps = new Pair(0l, new GuidePostsInfoBuilder()); guidePostsInfoWriterMap.put(cfKey, cachedGps); } } @@ -109,12 +109,6 @@ public long getMaxTimeStamp() { public void close() throws IOException { this.statsTable.close(); - for (ImmutableBytesPtr ptr : guidePostsInfoWriterMap.keySet()) { - GuidePostsInfoWriter guidePostWriter = guidePostsInfoWriterMap.get(ptr).getSecond(); - if (guidePostWriter != null) { - guidePostWriter.close(); - } - } } public void updateStatistic(Region region) { @@ -166,21 +160,21 @@ private void commitStats(List mutations) throws IOException { */ public void collectStatistics(final List results) { Map famMap = Maps.newHashMap(); - List rowTracker = null; + List rowTracker = null; if (cachedGps == null) { - rowTracker = new ArrayList(); + rowTracker = new ArrayList(); } for (Cell cell : results) { KeyValue kv = KeyValueUtil.ensureKeyValue(cell); maxTimeStamp = Math.max(maxTimeStamp, kv.getTimestamp()); - Pair gps; + Pair gps; if (cachedGps == null) { ImmutableBytesPtr cfKey = new ImmutableBytesPtr(kv.getFamilyArray(), kv.getFamilyOffset(), kv.getFamilyLength()); gps = guidePostsInfoWriterMap.get(cfKey); if (gps == null) { - gps = new Pair(0l, - new GuidePostsInfoWriter()); + gps = new Pair(0l, + new GuidePostsInfoBuilder()); guidePostsInfoWriterMap.put(cfKey, gps); } if (famMap.get(cfKey) == null) { @@ -196,13 +190,13 @@ public void collectStatistics(final List results) { if (byteCount >= guidepostDepth) { byte[] row = ByteUtil.copyKeyBytesIfNecessary( new ImmutableBytesWritable(kv.getRowArray(), kv.getRowOffset(), kv.getRowLength())); - if (gps.getSecond().writeGuidePosts(row, byteCount)) { + if (gps.getSecond().addGuidePosts(row, byteCount)) { gps.setFirst(0l); } } } if (cachedGps == null) { - for (GuidePostsInfoWriter s : rowTracker) { + for (GuidePostsInfoBuilder s : rowTracker) { s.incrementRowCount(); } } else { @@ -231,8 +225,8 @@ public void clear() { } public GuidePostsInfo getGuidePosts(ImmutableBytesPtr fam) { - Pair pair = guidePostsInfoWriterMap.get(fam); - if (pair != null) { return pair.getSecond().createGuidePostInfo(); } + Pair pair = guidePostsInfoWriterMap.get(fam); + if (pair != null) { return pair.getSecond().build(); } return null; } diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsUtil.java index 41a92c1a675..5b471041d30 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsUtil.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsUtil.java @@ -130,7 +130,7 @@ public static PTableStats readStatistics(HTableInterface statsHTable, byte[] tab s.addColumn(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES, QueryConstants.EMPTY_COLUMN_BYTES); ResultScanner scanner = null; long timeStamp = MetaDataProtocol.MIN_TABLE_TIMESTAMP; - TreeMap guidePostsInfoWriterPerCf = new TreeMap(Bytes.BYTES_COMPARATOR); + TreeMap guidePostsInfoWriterPerCf = new TreeMap(Bytes.BYTES_COMPARATOR); try { scanner = statsHTable.getScanner(s); Result result = null; @@ -172,12 +172,12 @@ public static PTableStats readStatistics(HTableInterface statsHTable, byte[] tab } if (cfName != null) { byte[] newGPStartKey = getGuidePostsInfoFromRowKey(tableNameBytes, cfName, result.getRow()); - GuidePostsInfoWriter guidePostsInfoWriter = guidePostsInfoWriterPerCf.get(cfName); + GuidePostsInfoBuilder guidePostsInfoWriter = guidePostsInfoWriterPerCf.get(cfName); if (guidePostsInfoWriter == null) { - guidePostsInfoWriter = new GuidePostsInfoWriter(); + guidePostsInfoWriter = new GuidePostsInfoBuilder(); guidePostsInfoWriterPerCf.put(cfName, guidePostsInfoWriter); } - guidePostsInfoWriter.writeGuidePosts(newGPStartKey, byteCount, rowCount); + guidePostsInfoWriter.addGuidePosts(newGPStartKey, byteCount, rowCount); } } if (!guidePostsInfoWriterPerCf.isEmpty()) { return new PTableStatsImpl( @@ -186,21 +186,15 @@ public static PTableStats readStatistics(HTableInterface statsHTable, byte[] tab if (scanner != null) { scanner.close(); } - for (byte[] cf : guidePostsInfoWriterPerCf.keySet()) { - GuidePostsInfoWriter guidePostsInfoWriter = guidePostsInfoWriterPerCf.get(cf); - if (guidePostsInfoWriter != null) { - guidePostsInfoWriter.close(); - } - } } return PTableStats.EMPTY_STATS; } private static SortedMap getGuidePostsPerCf( - TreeMap guidePostsWriterPerCf) { + TreeMap guidePostsWriterPerCf) { TreeMap guidePostsPerCf = new TreeMap(Bytes.BYTES_COMPARATOR); for (byte[] key : guidePostsWriterPerCf.keySet()) { - guidePostsPerCf.put(key, guidePostsWriterPerCf.get(key).createGuidePostInfo()); + guidePostsPerCf.put(key, guidePostsWriterPerCf.get(key).build()); } return guidePostsPerCf; } diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsWriter.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsWriter.java index 188434ac356..d03af7ab0ab 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsWriter.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsWriter.java @@ -49,7 +49,7 @@ import org.apache.phoenix.schema.types.PDate; import org.apache.phoenix.schema.types.PLong; import org.apache.phoenix.util.ByteUtil; -import org.apache.phoenix.util.CodecUtils; +import org.apache.phoenix.util.PrefixByteCodec; import org.apache.phoenix.util.PrefixByteDecoder; import org.apache.phoenix.util.ServerUtil; import org.apache.phoenix.util.TimeKeeper; @@ -166,7 +166,7 @@ public void addStats(StatisticsCollector tracker, ImmutableBytesPtr cfKey, List< } catch (EOFException e) { // Ignore as this signifies we're done } finally { - CodecUtils.close(stream); + PrefixByteCodec.close(stream); } } } diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/PrefixByteCodec.java b/phoenix-core/src/main/java/org/apache/phoenix/util/PrefixByteCodec.java index f4b330269e9..5f4a3441a9b 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/util/PrefixByteCodec.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/util/PrefixByteCodec.java @@ -31,7 +31,7 @@ import com.google.common.collect.Lists; -public class CodecUtils { +public class PrefixByteCodec { public static List decodeBytes(ImmutableBytesWritable encodedBytes, int maxLength) throws IOException { ByteArrayInputStream stream = new ByteArrayInputStream(encodedBytes.get(), encodedBytes.getOffset(), encodedBytes.getLength()); diff --git a/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanBigFilterTest.java b/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanBigFilterTest.java index 7c61ace7a0a..bb5f4084ad8 100644 --- a/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanBigFilterTest.java +++ b/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanBigFilterTest.java @@ -38,7 +38,7 @@ import org.apache.phoenix.schema.PTableImpl; import org.apache.phoenix.schema.PTableKey; import org.apache.phoenix.schema.stats.GuidePostsInfo; -import org.apache.phoenix.schema.stats.GuidePostsInfoWriter; +import org.apache.phoenix.schema.stats.GuidePostsInfoBuilder; import org.apache.phoenix.schema.stats.PTableStats; import org.apache.phoenix.util.PropertiesUtil; import org.apache.phoenix.util.ReadOnlyProps; @@ -647,12 +647,11 @@ private void testIntersect(byte[][] regionBoundaries, byte[][] guidePosts) throw stmt.execute(); final PTable table = conn.unwrap(PhoenixConnection.class).getTable(new PTableKey(null, "PERF.BIG_OLAP_DOC")); - GuidePostsInfoWriter gpWriter = new GuidePostsInfoWriter(); + GuidePostsInfoBuilder gpWriter = new GuidePostsInfoBuilder(); for (byte[] gp : guidePosts) { - gpWriter.writeGuidePosts(gp, 1000); + gpWriter.addGuidePosts(gp, 1000); } - GuidePostsInfo info = gpWriter.createGuidePostInfo(); - gpWriter.close(); + GuidePostsInfo info = gpWriter.build(); final SortedMap gpMap = Maps.newTreeMap(Bytes.BYTES_COMPARATOR); gpMap.put(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES, info); PTable tableWithStats = PTableImpl.makePTable(table, new PTableStats() { diff --git a/phoenix-core/src/test/java/org/apache/phoenix/util/PrefixByteEncoderDecoderTest.java b/phoenix-core/src/test/java/org/apache/phoenix/util/PrefixByteEncoderDecoderTest.java index ecb3efff94c..269ae1fed4a 100644 --- a/phoenix-core/src/test/java/org/apache/phoenix/util/PrefixByteEncoderDecoderTest.java +++ b/phoenix-core/src/test/java/org/apache/phoenix/util/PrefixByteEncoderDecoderTest.java @@ -53,9 +53,9 @@ public class PrefixByteEncoderDecoderTest { public void testEncode() throws IOException { List listOfBytes = Arrays.asList(Bytes.toBytes("aaaaa"), Bytes.toBytes("aaaabb")); ImmutableBytesWritable ptr = new ImmutableBytesWritable(); - int maxLength = CodecUtils.encodeBytes(listOfBytes, ptr); + int maxLength = PrefixByteCodec.encodeBytes(listOfBytes, ptr); assertEquals(6, maxLength); - TrustedByteArrayOutputStream stream = new TrustedByteArrayOutputStream(CodecUtils.calculateSize(listOfBytes)); + TrustedByteArrayOutputStream stream = new TrustedByteArrayOutputStream(PrefixByteCodec.calculateSize(listOfBytes)); DataOutput output = new DataOutputStream(stream); WritableUtils.writeVInt(output, 0); WritableUtils.writeVInt(output, 5); @@ -78,11 +78,11 @@ public void testEncodeDecodeWithNewBuffer() throws IOException { private void testEncodeDecode(boolean useSingleBuffer) throws IOException { ImmutableBytesWritable ptr = new ImmutableBytesWritable(); - int maxLength = CodecUtils.encodeBytes(guideposts, ptr); + int maxLength = PrefixByteCodec.encodeBytes(guideposts, ptr); int encodedSize = ptr.getLength(); - int unencodedSize = CodecUtils.calculateSize(guideposts); + int unencodedSize = PrefixByteCodec.calculateSize(guideposts); assertTrue(encodedSize < unencodedSize); - List listOfBytes = CodecUtils.decodeBytes(ptr, useSingleBuffer ? maxLength : -1); + List listOfBytes = PrefixByteCodec.decodeBytes(ptr, useSingleBuffer ? maxLength : -1); assertListByteArraysEquals(guideposts, listOfBytes); } From b711ba2e8babbb4635da33c959ecd167bb551795 Mon Sep 17 00:00:00 2001 From: Ankit Singhal Date: Mon, 18 Jan 2016 00:14:41 +0530 Subject: [PATCH 17/47] updated class level comments for GuidePostsInfoBuilder --- .../org/apache/phoenix/schema/stats/GuidePostsInfoBuilder.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsInfoBuilder.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsInfoBuilder.java index 11a1faaab7c..a8244da1ab3 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsInfoBuilder.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsInfoBuilder.java @@ -28,7 +28,7 @@ import org.apache.phoenix.util.TrustedByteArrayOutputStream; /* - * Writer to help in writing guidePosts and creating guidePostInfo. This is used when we are collecting stats or reading stats for a table. + * Builder to help in adding guidePosts and building guidePostInfo. This is used when we are collecting stats or reading stats for a table. */ public class GuidePostsInfoBuilder { From 59b336ec84b7cf5189bb2e67f07f9ef20da898d9 Mon Sep 17 00:00:00 2001 From: James Taylor Date: Sun, 17 Jan 2016 11:24:26 -0800 Subject: [PATCH 18/47] PHOENIX-2520 Create DDL property for metadata update frequency --- .../org/apache/phoenix/rpc/UpdateCacheIT.java | 92 +++++++++---- .../phoenix/rpc/UpdateCacheWithScnIT.java | 17 +-- phoenix-core/src/main/antlr3/PhoenixSQL.g | 3 +- .../apache/phoenix/compile/FromCompiler.java | 2 +- .../apache/phoenix/compile/JoinCompiler.java | 2 +- .../compile/TupleProjectionCompiler.java | 4 +- .../apache/phoenix/compile/UnionCompiler.java | 2 +- .../coprocessor/MetaDataEndpointImpl.java | 13 +- .../phoenix/coprocessor/MetaDataProtocol.java | 2 +- .../coprocessor/generated/PTableProtos.java | 104 ++++++++++++++- .../phoenix/jdbc/PhoenixConnection.java | 6 +- .../phoenix/jdbc/PhoenixDatabaseMetaData.java | 3 + .../query/ConnectionQueryServicesImpl.java | 43 +++--- .../ConnectionlessQueryServicesImpl.java | 4 +- .../DelegateConnectionQueryServices.java | 4 +- .../apache/phoenix/query/MetaDataMutated.java | 2 +- .../apache/phoenix/query/QueryConstants.java | 4 +- .../apache/phoenix/schema/DelegateTable.java | 6 + .../apache/phoenix/schema/MetaDataClient.java | 123 +++++++++++------ .../org/apache/phoenix/schema/PMetaData.java | 3 +- .../apache/phoenix/schema/PMetaDataImpl.java | 15 ++- .../org/apache/phoenix/schema/PTable.java | 1 + .../org/apache/phoenix/schema/PTableImpl.java | 65 ++++++--- .../org/apache/phoenix/schema/PTableRef.java | 17 ++- .../apache/phoenix/schema/TableProperty.java | 26 ++++ .../phoenix/execute/CorrelatePlanTest.java | 4 +- .../phoenix/schema/PMetaDataImplTest.java | 125 +++++++++++------- phoenix-protocol/src/main/PTable.proto | 1 + 28 files changed, 487 insertions(+), 206 deletions(-) diff --git a/phoenix-core/src/it/java/org/apache/phoenix/rpc/UpdateCacheIT.java b/phoenix-core/src/it/java/org/apache/phoenix/rpc/UpdateCacheIT.java index 13ed8aade34..20a3c48c9ee 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/rpc/UpdateCacheIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/rpc/UpdateCacheIT.java @@ -49,8 +49,8 @@ import org.apache.phoenix.util.PhoenixRuntime; import org.apache.phoenix.util.PropertiesUtil; import org.apache.phoenix.util.ReadOnlyProps; +import org.apache.phoenix.util.SchemaUtil; import org.apache.phoenix.util.TestUtil; -import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; import org.mockito.Mockito; @@ -73,48 +73,86 @@ public static void doSetup() throws Exception { setUpTestDriver(new ReadOnlyProps(props.entrySet().iterator())); } - @Before - public void setUp() throws SQLException { - ensureTableCreated(getUrl(), MUTABLE_INDEX_DATA_TABLE); - ensureTableCreated(getUrl(), TRANSACTIONAL_DATA_TABLE); - } - - private static void setupSystemTable(Long scn) throws SQLException { + private static void setupSystemTable(String fullTableName) throws SQLException { Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); - if (scn != null) { - props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(scn)); - } try (Connection conn = DriverManager.getConnection(getUrl(), props)) { conn.createStatement().execute( - "create table " + QueryConstants.SYSTEM_SCHEMA_NAME + QueryConstants.NAME_SEPARATOR + MUTABLE_INDEX_DATA_TABLE + TEST_TABLE_SCHEMA); + "create table " + fullTableName + TEST_TABLE_SCHEMA); } } @Test public void testUpdateCacheForTxnTable() throws Exception { - helpTestUpdateCache(true, false, null); + String fullTableName = INDEX_DATA_SCHEMA + QueryConstants.NAME_SEPARATOR + TRANSACTIONAL_DATA_TABLE; + ensureTableCreated(getUrl(), TRANSACTIONAL_DATA_TABLE); + helpTestUpdateCache(fullTableName, null, new int[] {1, 1}); } @Test public void testUpdateCacheForNonTxnTable() throws Exception { - helpTestUpdateCache(false, false, null); + String fullTableName = INDEX_DATA_SCHEMA + QueryConstants.NAME_SEPARATOR + MUTABLE_INDEX_DATA_TABLE; + ensureTableCreated(getUrl(), MUTABLE_INDEX_DATA_TABLE); + helpTestUpdateCache(fullTableName, null, new int[] {1, 3}); } @Test public void testUpdateCacheForNonTxnSystemTable() throws Exception { - helpTestUpdateCache(false, true, null); + String fullTableName = QueryConstants.SYSTEM_SCHEMA_NAME + QueryConstants.NAME_SEPARATOR + MUTABLE_INDEX_DATA_TABLE; + setupSystemTable(fullTableName); + helpTestUpdateCache(fullTableName, null, new int[] {0, 0}); + } + + @Test + public void testUpdateCacheForNeverUpdatedTable() throws Exception { + String fullTableName = INDEX_DATA_SCHEMA + QueryConstants.NAME_SEPARATOR + MUTABLE_INDEX_DATA_TABLE; + ensureTableCreated(getUrl(), MUTABLE_INDEX_DATA_TABLE); + Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); + try (Connection conn = DriverManager.getConnection(getUrl(), props)) { + conn.createStatement().execute( + "alter table " + fullTableName + " SET UPDATE_CACHE_FREQUENCY=NEVER"); + } + helpTestUpdateCache(fullTableName, null, new int[] {0, 0}); + } + + @Test + public void testUpdateCacheForAlwaysUpdatedTable() throws Exception { + String fullTableName = INDEX_DATA_SCHEMA + QueryConstants.NAME_SEPARATOR + MUTABLE_INDEX_DATA_TABLE; + Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); + try (Connection conn = DriverManager.getConnection(getUrl(), props)) { + conn.createStatement().execute("CREATE TABLE " + fullTableName + TEST_TABLE_SCHEMA + " UPDATE_CACHE_FREQUENCY=always"); + } + helpTestUpdateCache(fullTableName, null, new int[] {1, 3}); + } + + @Test + public void testUpdateCacheForTimeLimitedUpdateTable() throws Exception { + String fullTableName = INDEX_DATA_SCHEMA + QueryConstants.NAME_SEPARATOR + MUTABLE_INDEX_DATA_TABLE; + Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); + try (Connection conn = DriverManager.getConnection(getUrl(), props)) { + conn.createStatement().execute("CREATE TABLE " + fullTableName + TEST_TABLE_SCHEMA + " UPDATE_CACHE_FREQUENCY=" + 10000); + } + helpTestUpdateCache(fullTableName, null, new int[] {0, 0}); + Thread.sleep(10000); + helpTestUpdateCache(fullTableName, null, new int[] {1, 0}); + } + + @Test + public void testUpdateCacheForChangingUpdateTable() throws Exception { + String fullTableName = INDEX_DATA_SCHEMA + QueryConstants.NAME_SEPARATOR + MUTABLE_INDEX_DATA_TABLE; + Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); + try (Connection conn = DriverManager.getConnection(getUrl(), props)) { + conn.createStatement().execute("CREATE TABLE " + fullTableName + TEST_TABLE_SCHEMA + " UPDATE_CACHE_FREQUENCY=never"); + } + helpTestUpdateCache(fullTableName, null, new int[] {0, 0}); + try (Connection conn = DriverManager.getConnection(getUrl(), props)) { + conn.createStatement().execute("ALTER TABLE " + fullTableName + " SET UPDATE_CACHE_FREQUENCY=ALWAYS"); + } + helpTestUpdateCache(fullTableName, null, new int[] {1, 3}); } - public static void helpTestUpdateCache(boolean isTransactional, boolean isSystem, Long scn) throws Exception { - String tableName = isTransactional ? TRANSACTIONAL_DATA_TABLE : MUTABLE_INDEX_DATA_TABLE; - String schemaName; - if (isSystem) { - setupSystemTable(scn); - schemaName = QueryConstants.SYSTEM_SCHEMA_NAME; - } else { - schemaName = INDEX_DATA_SCHEMA; - } - String fullTableName = schemaName + QueryConstants.NAME_SEPARATOR + tableName; + public static void helpTestUpdateCache(String fullTableName, Long scn, int[] expectedRPCs) throws Exception { + String tableName = SchemaUtil.getTableNameFromFullName(fullTableName); + String schemaName = SchemaUtil.getSchemaNameFromFullName(fullTableName); String selectSql = "SELECT * FROM "+fullTableName; // use a spyed ConnectionQueryServices so we can verify calls to getTable ConnectionQueryServices connectionQueryServices = Mockito.spy(driver.getConnectionQueryServices(getUrl(), PropertiesUtil.deepCopy(TEST_PROPERTIES))); @@ -136,7 +174,7 @@ public static void helpTestUpdateCache(boolean isTransactional, boolean isSystem TestUtil.setRowKeyColumns(stmt, 3); stmt.execute(); conn.commit(); - int numUpsertRpcs = isSystem ? 0 : 1; + int numUpsertRpcs = expectedRPCs[0]; // verify only 0 or 1 rpc to fetch table metadata, verify(connectionQueryServices, times(numUpsertRpcs)).getTable((PName)isNull(), eq(PVarchar.INSTANCE.toBytes(schemaName)), eq(PVarchar.INSTANCE.toBytes(tableName)), anyLong(), anyLong()); reset(connectionQueryServices); @@ -169,7 +207,7 @@ public static void helpTestUpdateCache(boolean isTransactional, boolean isSystem // for non-transactional tables with a scn : verify *only* one rpc occurs // for transactional tables : verify *only* one rpc occurs // for non-transactional, system tables : verify no rpc occurs - int numRpcs = isSystem ? 0 : (isTransactional || scn!=null ? 1 : 3); + int numRpcs = expectedRPCs[1]; verify(connectionQueryServices, times(numRpcs)).getTable((PName)isNull(), eq(PVarchar.INSTANCE.toBytes(schemaName)), eq(PVarchar.INSTANCE.toBytes(tableName)), anyLong(), anyLong()); } finally { diff --git a/phoenix-core/src/it/java/org/apache/phoenix/rpc/UpdateCacheWithScnIT.java b/phoenix-core/src/it/java/org/apache/phoenix/rpc/UpdateCacheWithScnIT.java index 5ff2fb0d9b6..04f751b2407 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/rpc/UpdateCacheWithScnIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/rpc/UpdateCacheWithScnIT.java @@ -17,25 +17,22 @@ */ package org.apache.phoenix.rpc; +import static org.apache.phoenix.util.TestUtil.INDEX_DATA_SCHEMA; import static org.apache.phoenix.util.TestUtil.MUTABLE_INDEX_DATA_TABLE; import org.apache.phoenix.end2end.BaseClientManagedTimeIT; -import org.junit.Before; +import org.apache.phoenix.query.QueryConstants; import org.junit.Test; public class UpdateCacheWithScnIT extends BaseClientManagedTimeIT { - protected long ts; - - @Before - public void initTable() throws Exception { - ts = nextTimestamp(); - ensureTableCreated(getUrl(), MUTABLE_INDEX_DATA_TABLE, ts); - } - @Test public void testUpdateCacheWithScn() throws Exception { - UpdateCacheIT.helpTestUpdateCache(false, false, ts+2); + long ts = nextTimestamp(); + String fullTableName = INDEX_DATA_SCHEMA + QueryConstants.NAME_SEPARATOR + MUTABLE_INDEX_DATA_TABLE; + ensureTableCreated(getUrl(), MUTABLE_INDEX_DATA_TABLE, ts); + // FIXME: given that the scn is advancing in the test, why aren't there more RPCs? + UpdateCacheIT.helpTestUpdateCache(fullTableName, ts+2, new int[] {1, 1}); } } diff --git a/phoenix-core/src/main/antlr3/PhoenixSQL.g b/phoenix-core/src/main/antlr3/PhoenixSQL.g index 18a153ed0d0..23f7e8f8fc2 100644 --- a/phoenix-core/src/main/antlr3/PhoenixSQL.g +++ b/phoenix-core/src/main/antlr3/PhoenixSQL.g @@ -509,7 +509,8 @@ fam_prop_name returns [PropertyName ret] ; prop_value returns [Object ret] - : l=literal { $ret = l.getValue(); } + : v=identifier { $ret = v; } + | l=literal { $ret = l.getValue(); } ; column_name returns [ColumnName ret] diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/FromCompiler.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/FromCompiler.java index 0828b94c1af..9b2c460d2b2 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/compile/FromCompiler.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/FromCompiler.java @@ -652,7 +652,7 @@ public Void visit(DerivedTableNode subselectNode) throws SQLException { PTableType.SUBQUERY, null, MetaDataProtocol.MIN_TABLE_TIMESTAMP, PTable.INITIAL_SEQ_NUM, null, null, columns, null, null, Collections.emptyList(), false, Collections.emptyList(), null, null, false, false, false, null, - null, null, false, false); + null, null, false, false, 0); String alias = subselectNode.getAlias(); TableRef tableRef = new TableRef(alias, t, MetaDataProtocol.MIN_TABLE_TIMESTAMP, false); diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/JoinCompiler.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/JoinCompiler.java index b55e4aad720..b64b9b7e7a2 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/compile/JoinCompiler.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/JoinCompiler.java @@ -1302,7 +1302,7 @@ public static PTable joinProjectedTables(PTable left, PTable right, JoinType typ left.getBucketNum(), merged,left.getParentSchemaName(), left.getParentTableName(), left.getIndexes(), left.isImmutableRows(), Collections.emptyList(), null, null, PTable.DEFAULT_DISABLE_WAL, left.isMultiTenant(), left.getStoreNulls(), left.getViewType(), left.getViewIndexId(), left.getIndexType(), - left.rowKeyOrderOptimizable(), left.isTransactional()); + left.rowKeyOrderOptimizable(), left.isTransactional(), left.getUpdateCacheFrequency()); } } diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/TupleProjectionCompiler.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/TupleProjectionCompiler.java index 551b05cb2ad..0fc6d74902f 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/compile/TupleProjectionCompiler.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/TupleProjectionCompiler.java @@ -152,7 +152,7 @@ public static PTable createProjectedTable(SelectStatement select, StatementConte table.getBucketNum(), projectedColumns, table.getParentSchemaName(), table.getParentName(), table.getIndexes(), table.isImmutableRows(), Collections.emptyList(), null, null, table.isWALDisabled(), table.isMultiTenant(), table.getStoreNulls(), table.getViewType(), table.getViewIndexId(), - table.getIndexType(), table.rowKeyOrderOptimizable(), table.isTransactional()); + table.getIndexType(), table.rowKeyOrderOptimizable(), table.isTransactional(), table.getUpdateCacheFrequency()); } public static PTable createProjectedTable(TableRef tableRef, List sourceColumnRefs, boolean retainPKColumns) throws SQLException { @@ -179,7 +179,7 @@ public static PTable createProjectedTable(TableRef tableRef, List sou retainPKColumns ? table.getBucketNum() : null, projectedColumns, null, null, Collections.emptyList(), table.isImmutableRows(), Collections.emptyList(), null, null, table.isWALDisabled(), table.isMultiTenant(), table.getStoreNulls(), table.getViewType(), table.getViewIndexId(), - null, table.rowKeyOrderOptimizable(), table.isTransactional()); + null, table.rowKeyOrderOptimizable(), table.isTransactional(), table.getUpdateCacheFrequency()); } // For extracting column references from single select statement diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/UnionCompiler.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/UnionCompiler.java index 298303dbdd4..3bc1e374a9a 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/compile/UnionCompiler.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/UnionCompiler.java @@ -82,7 +82,7 @@ public static TableRef contructSchemaTable(PhoenixStatement statement, QueryPlan PTable tempTable = PTableImpl.makePTable(statement.getConnection().getTenantId(), UNION_SCHEMA_NAME, UNION_TABLE_NAME, PTableType.SUBQUERY, null, HConstants.LATEST_TIMESTAMP, scn == null ? HConstants.LATEST_TIMESTAMP : scn, null, null, projectedColumns, null, null, null, - true, null, null, null, true, true, true, null, null, null, false, false); + true, null, null, null, true, true, true, null, null, null, false, false, 0); TableRef tableRef = new TableRef(null, tempTable, 0, false); return tableRef; } diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java index 8c905bad250..9887e7bec13 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java @@ -62,6 +62,7 @@ import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TENANT_ID_INDEX; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TRANSACTIONAL_BYTES; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TYPE_BYTES; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.UPDATE_CACHE_FREQUENCY_BYTES; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_CONSTANT_BYTES; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_INDEX_ID_BYTES; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_STATEMENT_BYTES; @@ -242,6 +243,7 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements Coprocesso private static final KeyValue BASE_COLUMN_COUNT_KV = createFirstOnRow(ByteUtil.EMPTY_BYTE_ARRAY, TABLE_FAMILY_BYTES, PhoenixDatabaseMetaData.BASE_COLUMN_COUNT_BYTES); private static final KeyValue ROW_KEY_ORDER_OPTIMIZABLE_KV = createFirstOnRow(ByteUtil.EMPTY_BYTE_ARRAY, TABLE_FAMILY_BYTES, ROW_KEY_ORDER_OPTIMIZABLE_BYTES); private static final KeyValue TRANSACTIONAL_KV = createFirstOnRow(ByteUtil.EMPTY_BYTE_ARRAY, TABLE_FAMILY_BYTES, TRANSACTIONAL_BYTES); + private static final KeyValue UPDATE_CACHE_FREQUENCY_KV = createFirstOnRow(ByteUtil.EMPTY_BYTE_ARRAY, TABLE_FAMILY_BYTES, UPDATE_CACHE_FREQUENCY_BYTES); private static final List TABLE_KV_COLUMNS = Arrays.asList( EMPTY_KEYVALUE_KV, @@ -264,7 +266,8 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements Coprocesso STORE_NULLS_KV, BASE_COLUMN_COUNT_KV, ROW_KEY_ORDER_OPTIMIZABLE_KV, - TRANSACTIONAL_KV + TRANSACTIONAL_KV, + UPDATE_CACHE_FREQUENCY_KV ); static { Collections.sort(TABLE_KV_COLUMNS, KeyValue.COMPARATOR); @@ -289,6 +292,7 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements Coprocesso private static final int BASE_COLUMN_COUNT_INDEX = TABLE_KV_COLUMNS.indexOf(BASE_COLUMN_COUNT_KV); private static final int ROW_KEY_ORDER_OPTIMIZABLE_INDEX = TABLE_KV_COLUMNS.indexOf(ROW_KEY_ORDER_OPTIMIZABLE_KV); private static final int TRANSACTIONAL_INDEX = TABLE_KV_COLUMNS.indexOf(TRANSACTIONAL_KV); + private static final int UPDATE_CACHE_FREQUENCY_INDEX = TABLE_KV_COLUMNS.indexOf(UPDATE_CACHE_FREQUENCY_KV); // KeyValues for Column private static final KeyValue DECIMAL_DIGITS_KV = createFirstOnRow(ByteUtil.EMPTY_BYTE_ARRAY, TABLE_FAMILY_BYTES, DECIMAL_DIGITS_BYTES); @@ -806,6 +810,10 @@ private PTable getTable(RegionScanner scanner, long clientTimeStamp, long tableT baseColumnCountKv.getValueOffset(), SortOrder.getDefault()); Cell rowKeyOrderOptimizableKv = tableKeyValues[ROW_KEY_ORDER_OPTIMIZABLE_INDEX]; boolean rowKeyOrderOptimizable = rowKeyOrderOptimizableKv == null ? false : Boolean.TRUE.equals(PBoolean.INSTANCE.toObject(rowKeyOrderOptimizableKv.getValueArray(), rowKeyOrderOptimizableKv.getValueOffset(), rowKeyOrderOptimizableKv.getValueLength())); + Cell updateCacheFrequencyKv = tableKeyValues[UPDATE_CACHE_FREQUENCY_INDEX]; + long updateCacheFrequency = updateCacheFrequencyKv == null ? 0 : + PLong.INSTANCE.getCodec().decodeLong(updateCacheFrequencyKv.getValueArray(), + updateCacheFrequencyKv.getValueOffset(), SortOrder.getDefault()); List columns = Lists.newArrayListWithExpectedSize(columnCount); List indexes = new ArrayList(); @@ -850,7 +858,8 @@ private PTable getTable(RegionScanner scanner, long clientTimeStamp, long tableT return PTableImpl.makePTable(tenantId, schemaName, tableName, tableType, indexState, timeStamp, tableSeqNum, pkName, saltBucketNum, columns, tableType == INDEX ? schemaName : null, tableType == INDEX ? dataTableName : null, indexes, isImmutableRows, physicalTables, defaultFamilyName, viewStatement, - disableWAL, multiTenant, storeNulls, viewType, viewIndexId, indexType, stats, baseColumnCount, rowKeyOrderOptimizable, transactional); + disableWAL, multiTenant, storeNulls, viewType, viewIndexId, indexType, rowKeyOrderOptimizable, transactional, updateCacheFrequency, + stats, baseColumnCount); } private PFunction getFunction(RegionScanner scanner, final boolean isReplace, long clientTimeStamp, List deleteMutationsForReplace) diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataProtocol.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataProtocol.java index b7b936e969a..a704e22d0ea 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataProtocol.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataProtocol.java @@ -73,7 +73,7 @@ public abstract class MetaDataProtocol extends MetaDataService { public static final long MIN_SYSTEM_TABLE_TIMESTAMP_4_3_0 = MIN_TABLE_TIMESTAMP + 7; public static final long MIN_SYSTEM_TABLE_TIMESTAMP_4_5_0 = MIN_TABLE_TIMESTAMP + 8; public static final long MIN_SYSTEM_TABLE_TIMESTAMP_4_6_0 = MIN_TABLE_TIMESTAMP + 9; - public static final long MIN_SYSTEM_TABLE_TIMESTAMP_4_7_0 = MIN_TABLE_TIMESTAMP + 11; + public static final long MIN_SYSTEM_TABLE_TIMESTAMP_4_7_0 = MIN_TABLE_TIMESTAMP + 13; // MIN_SYSTEM_TABLE_TIMESTAMP needs to be set to the max of all the MIN_SYSTEM_TABLE_TIMESTAMP_* constants public static final long MIN_SYSTEM_TABLE_TIMESTAMP = MIN_SYSTEM_TABLE_TIMESTAMP_4_7_0; // TODO: pare this down to minimum, as we don't need duplicates for both table and column errors, nor should we need diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/PTableProtos.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/PTableProtos.java index 7e71cd9fa8a..be8d7e2b93a 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/PTableProtos.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/PTableProtos.java @@ -3228,6 +3228,16 @@ org.apache.phoenix.coprocessor.generated.PTableProtos.PTableStatsOrBuilder getGu * optional bool transactional = 27; */ boolean getTransactional(); + + // optional int64 updateCacheFrequency = 28; + /** + * optional int64 updateCacheFrequency = 28; + */ + boolean hasUpdateCacheFrequency(); + /** + * optional int64 updateCacheFrequency = 28; + */ + long getUpdateCacheFrequency(); } /** * Protobuf type {@code PTable} @@ -3433,6 +3443,11 @@ private PTable( transactional_ = input.readBool(); break; } + case 224: { + bitField0_ |= 0x00800000; + updateCacheFrequency_ = input.readInt64(); + break; + } } } } catch (com.google.protobuf.InvalidProtocolBufferException e) { @@ -4011,6 +4026,22 @@ public boolean getTransactional() { return transactional_; } + // optional int64 updateCacheFrequency = 28; + public static final int UPDATECACHEFREQUENCY_FIELD_NUMBER = 28; + private long updateCacheFrequency_; + /** + * optional int64 updateCacheFrequency = 28; + */ + public boolean hasUpdateCacheFrequency() { + return ((bitField0_ & 0x00800000) == 0x00800000); + } + /** + * optional int64 updateCacheFrequency = 28; + */ + public long getUpdateCacheFrequency() { + return updateCacheFrequency_; + } + private void initFields() { schemaNameBytes_ = com.google.protobuf.ByteString.EMPTY; tableNameBytes_ = com.google.protobuf.ByteString.EMPTY; @@ -4039,6 +4070,7 @@ private void initFields() { baseColumnCount_ = 0; rowKeyOrderOptimizable_ = false; transactional_ = false; + updateCacheFrequency_ = 0L; } private byte memoizedIsInitialized = -1; public final boolean isInitialized() { @@ -4187,6 +4219,9 @@ public void writeTo(com.google.protobuf.CodedOutputStream output) if (((bitField0_ & 0x00400000) == 0x00400000)) { output.writeBool(27, transactional_); } + if (((bitField0_ & 0x00800000) == 0x00800000)) { + output.writeInt64(28, updateCacheFrequency_); + } getUnknownFields().writeTo(output); } @@ -4309,6 +4344,10 @@ public int getSerializedSize() { size += com.google.protobuf.CodedOutputStream .computeBoolSize(27, transactional_); } + if (((bitField0_ & 0x00800000) == 0x00800000)) { + size += com.google.protobuf.CodedOutputStream + .computeInt64Size(28, updateCacheFrequency_); + } size += getUnknownFields().getSerializedSize(); memoizedSerializedSize = size; return size; @@ -4455,6 +4494,11 @@ public boolean equals(final java.lang.Object obj) { result = result && (getTransactional() == other.getTransactional()); } + result = result && (hasUpdateCacheFrequency() == other.hasUpdateCacheFrequency()); + if (hasUpdateCacheFrequency()) { + result = result && (getUpdateCacheFrequency() + == other.getUpdateCacheFrequency()); + } result = result && getUnknownFields().equals(other.getUnknownFields()); return result; @@ -4576,6 +4620,10 @@ public int hashCode() { hash = (37 * hash) + TRANSACTIONAL_FIELD_NUMBER; hash = (53 * hash) + hashBoolean(getTransactional()); } + if (hasUpdateCacheFrequency()) { + hash = (37 * hash) + UPDATECACHEFREQUENCY_FIELD_NUMBER; + hash = (53 * hash) + hashLong(getUpdateCacheFrequency()); + } hash = (29 * hash) + getUnknownFields().hashCode(); memoizedHashCode = hash; return hash; @@ -4754,6 +4802,8 @@ public Builder clear() { bitField0_ = (bitField0_ & ~0x02000000); transactional_ = false; bitField0_ = (bitField0_ & ~0x04000000); + updateCacheFrequency_ = 0L; + bitField0_ = (bitField0_ & ~0x08000000); return this; } @@ -4906,6 +4956,10 @@ public org.apache.phoenix.coprocessor.generated.PTableProtos.PTable buildPartial to_bitField0_ |= 0x00400000; } result.transactional_ = transactional_; + if (((from_bitField0_ & 0x08000000) == 0x08000000)) { + to_bitField0_ |= 0x00800000; + } + result.updateCacheFrequency_ = updateCacheFrequency_; result.bitField0_ = to_bitField0_; onBuilt(); return result; @@ -5081,6 +5135,9 @@ public Builder mergeFrom(org.apache.phoenix.coprocessor.generated.PTableProtos.P if (other.hasTransactional()) { setTransactional(other.getTransactional()); } + if (other.hasUpdateCacheFrequency()) { + setUpdateCacheFrequency(other.getUpdateCacheFrequency()); + } this.mergeUnknownFields(other.getUnknownFields()); return this; } @@ -6784,6 +6841,39 @@ public Builder clearTransactional() { return this; } + // optional int64 updateCacheFrequency = 28; + private long updateCacheFrequency_ ; + /** + * optional int64 updateCacheFrequency = 28; + */ + public boolean hasUpdateCacheFrequency() { + return ((bitField0_ & 0x08000000) == 0x08000000); + } + /** + * optional int64 updateCacheFrequency = 28; + */ + public long getUpdateCacheFrequency() { + return updateCacheFrequency_; + } + /** + * optional int64 updateCacheFrequency = 28; + */ + public Builder setUpdateCacheFrequency(long value) { + bitField0_ |= 0x08000000; + updateCacheFrequency_ = value; + onChanged(); + return this; + } + /** + * optional int64 updateCacheFrequency = 28; + */ + public Builder clearUpdateCacheFrequency() { + bitField0_ = (bitField0_ & ~0x08000000); + updateCacheFrequency_ = 0L; + onChanged(); + return this; + } + // @@protoc_insertion_point(builder_scope:PTable) } @@ -6830,7 +6920,7 @@ public Builder clearTransactional() { "leStats\022\013\n\003key\030\001 \002(\014\022\016\n\006values\030\002 \003(\014\022\033\n\023" + "guidePostsByteCount\030\003 \001(\003\022\025\n\rkeyBytesCou", "nt\030\004 \001(\003\022\027\n\017guidePostsCount\030\005 \001(\005\022!\n\013pGu" + - "idePosts\030\006 \001(\0132\014.PGuidePosts\"\206\005\n\006PTable\022" + + "idePosts\030\006 \001(\0132\014.PGuidePosts\"\244\005\n\006PTable\022" + "\027\n\017schemaNameBytes\030\001 \002(\014\022\026\n\016tableNameByt" + "es\030\002 \002(\014\022\036\n\ttableType\030\003 \002(\0162\013.PTableType" + "\022\022\n\nindexState\030\004 \001(\t\022\026\n\016sequenceNumber\030\005" + @@ -6846,11 +6936,11 @@ public Builder clearTransactional() { "wIndexId\030\025 \001(\005\022\021\n\tindexType\030\026 \001(\014\022\026\n\016sta" + "tsTimeStamp\030\027 \001(\003\022\022\n\nstoreNulls\030\030 \001(\010\022\027\n" + "\017baseColumnCount\030\031 \001(\005\022\036\n\026rowKeyOrderOpt" + - "imizable\030\032 \001(\010\022\025\n\rtransactional\030\033 \001(\010*A\n" + - "\nPTableType\022\n\n\006SYSTEM\020\000\022\010\n\004USER\020\001\022\010\n\004VIE" + - "W\020\002\022\t\n\005INDEX\020\003\022\010\n\004JOIN\020\004B@\n(org.apache.p", - "hoenix.coprocessor.generatedB\014PTableProt" + - "osH\001\210\001\001\240\001\001" + "imizable\030\032 \001(\010\022\025\n\rtransactional\030\033 \001(\010\022\034\n" + + "\024updateCacheFrequency\030\034 \001(\003*A\n\nPTableTyp" + + "e\022\n\n\006SYSTEM\020\000\022\010\n\004USER\020\001\022\010\n\004VIEW\020\002\022\t\n\005IND", + "EX\020\003\022\010\n\004JOIN\020\004B@\n(org.apache.phoenix.cop" + + "rocessor.generatedB\014PTableProtosH\001\210\001\001\240\001\001" }; com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner = new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() { @@ -6874,7 +6964,7 @@ public com.google.protobuf.ExtensionRegistry assignDescriptors( internal_static_PTable_fieldAccessorTable = new com.google.protobuf.GeneratedMessage.FieldAccessorTable( internal_static_PTable_descriptor, - new java.lang.String[] { "SchemaNameBytes", "TableNameBytes", "TableType", "IndexState", "SequenceNumber", "TimeStamp", "PkNameBytes", "BucketNum", "Columns", "Indexes", "IsImmutableRows", "GuidePosts", "DataTableNameBytes", "DefaultFamilyName", "DisableWAL", "MultiTenant", "ViewType", "ViewStatement", "PhysicalNames", "TenantId", "ViewIndexId", "IndexType", "StatsTimeStamp", "StoreNulls", "BaseColumnCount", "RowKeyOrderOptimizable", "Transactional", }); + new java.lang.String[] { "SchemaNameBytes", "TableNameBytes", "TableType", "IndexState", "SequenceNumber", "TimeStamp", "PkNameBytes", "BucketNum", "Columns", "Indexes", "IsImmutableRows", "GuidePosts", "DataTableNameBytes", "DefaultFamilyName", "DisableWAL", "MultiTenant", "ViewType", "ViewStatement", "PhysicalNames", "TenantId", "ViewIndexId", "IndexType", "StatsTimeStamp", "StoreNulls", "BaseColumnCount", "RowKeyOrderOptimizable", "Transactional", "UpdateCacheFrequency", }); return null; } }; diff --git a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixConnection.java b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixConnection.java index 208e874b81e..82bf31ac53f 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixConnection.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixConnection.java @@ -913,11 +913,11 @@ public PMetaData addFunction(PFunction function) throws SQLException { } @Override - public PMetaData addColumn(PName tenantId, String tableName, List columns, long tableTimeStamp, long tableSeqNum, boolean isImmutableRows, boolean isWalDisabled, boolean isMultitenant, boolean storeNulls, boolean isTransactional, long resolvedTime) + public PMetaData addColumn(PName tenantId, String tableName, List columns, long tableTimeStamp, long tableSeqNum, boolean isImmutableRows, boolean isWalDisabled, boolean isMultitenant, boolean storeNulls, boolean isTransactional, long updateCacheFrequency, long resolvedTime) throws SQLException { - metaData = metaData.addColumn(tenantId, tableName, columns, tableTimeStamp, tableSeqNum, isImmutableRows, isWalDisabled, isMultitenant, storeNulls, isTransactional, resolvedTime); + metaData = metaData.addColumn(tenantId, tableName, columns, tableTimeStamp, tableSeqNum, isImmutableRows, isWalDisabled, isMultitenant, storeNulls, isTransactional, updateCacheFrequency, resolvedTime); //Cascade through to connectionQueryServices too - getQueryServices().addColumn(tenantId, tableName, columns, tableTimeStamp, tableSeqNum, isImmutableRows, isWalDisabled, isMultitenant, storeNulls, isTransactional, resolvedTime); + getQueryServices().addColumn(tenantId, tableName, columns, tableTimeStamp, tableSeqNum, isImmutableRows, isWalDisabled, isMultitenant, storeNulls, isTransactional, updateCacheFrequency, resolvedTime); return metaData; } diff --git a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java index 869ba19b12c..fabd9498cfb 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java @@ -285,6 +285,9 @@ public class PhoenixDatabaseMetaData implements DatabaseMetaData { public static final String TRANSACTIONAL = "TRANSACTIONAL"; public static final byte[] TRANSACTIONAL_BYTES = Bytes.toBytes(TRANSACTIONAL); + public static final String UPDATE_CACHE_FREQUENCY = "UPDATE_CACHE_FREQUENCY"; + public static final byte[] UPDATE_CACHE_FREQUENCY_BYTES = Bytes.toBytes(UPDATE_CACHE_FREQUENCY); + public static final String ASYNC_CREATED_DATE = "ASYNC_CREATED_DATE"; private final PhoenixConnection connection; diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java index 8ef31617765..a246e63c323 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java @@ -630,12 +630,14 @@ private PMetaData metaDataMutated(PName tenantId, String tableName, long tableSe } @Override - public PMetaData addColumn(final PName tenantId, final String tableName, final List columns, final long tableTimeStamp, final long tableSeqNum, final boolean isImmutableRows, final boolean isWalDisabled, final boolean isMultitenant, final boolean storeNulls, final boolean isTransactional, final long resolvedTime) throws SQLException { + public PMetaData addColumn(final PName tenantId, final String tableName, final List columns, final long tableTimeStamp, + final long tableSeqNum, final boolean isImmutableRows, final boolean isWalDisabled, final boolean isMultitenant, + final boolean storeNulls, final boolean isTransactional, final long updateCacheFrequency, final long resolvedTime) throws SQLException { return metaDataMutated(tenantId, tableName, tableSeqNum, new Mutator() { @Override public PMetaData mutate(PMetaData metaData) throws SQLException { try { - return metaData.addColumn(tenantId, tableName, columns, tableTimeStamp, tableSeqNum, isImmutableRows, isWalDisabled, isMultitenant, storeNulls, isTransactional, resolvedTime); + return metaData.addColumn(tenantId, tableName, columns, tableTimeStamp, tableSeqNum, isImmutableRows, isWalDisabled, isMultitenant, storeNulls, isTransactional, updateCacheFrequency, resolvedTime); } catch (TableNotFoundException e) { // The DROP TABLE may have been processed first, so just ignore. return metaData; @@ -2410,17 +2412,20 @@ public Void call() throws Exception { } if (currentServerSideTableTimeStamp < MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_6_0) { columnsToAdd = PhoenixDatabaseMetaData.IS_ROW_TIMESTAMP + " " + PBoolean.INSTANCE.getSqlTypeName(); - metaConnection = addColumn(metaConnection, PhoenixDatabaseMetaData.SYSTEM_CATALOG, - MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_6_0, columnsToAdd, false); + metaConnection = addColumnsIfNotExists(metaConnection, PhoenixDatabaseMetaData.SYSTEM_CATALOG, + MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_6_0, columnsToAdd); } if(currentServerSideTableTimeStamp < MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_7_0) { - columnsToAdd = PhoenixDatabaseMetaData.TRANSACTIONAL + " " + PBoolean.INSTANCE.getSqlTypeName(); - metaConnection = addColumn(metaConnection, PhoenixDatabaseMetaData.SYSTEM_CATALOG, - MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_7_0, columnsToAdd, false); - // Drop old stats table so that new stats - // table + // Add these columns one at a time, each with different timestamps so that if folks have + // run the upgrade code already for a snapshot, we'll still enter this block (and do the + // parts we haven't yet done). + metaConnection = addColumnsIfNotExists(metaConnection, PhoenixDatabaseMetaData.SYSTEM_CATALOG, MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_7_0 - 2, + PhoenixDatabaseMetaData.TRANSACTIONAL + " " + PBoolean.INSTANCE.getSqlTypeName()); + metaConnection = addColumnsIfNotExists(metaConnection, PhoenixDatabaseMetaData.SYSTEM_CATALOG, MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_7_0 - 1, + PhoenixDatabaseMetaData.UPDATE_CACHE_FREQUENCY + " " + PLong.INSTANCE.getSqlTypeName()); + // Drop old stats table so that new stats table is created metaConnection = dropStatsTable(metaConnection, - MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_7_0 - 1); + MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_7_0); } } @@ -2525,7 +2530,7 @@ public Void call() throws Exception { private PhoenixConnection dropStatsTable(PhoenixConnection oldMetaConnection, long timestamp) throws SQLException, IOException { Properties props = PropertiesUtil.deepCopy(oldMetaConnection.getClientInfo()); - props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(timestamp)); + props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(timestamp-1)); PhoenixConnection metaConnection = new PhoenixConnection(oldMetaConnection, this, props); SQLException sqlE = null; boolean wasCommit = metaConnection.getAutoCommit(); @@ -2555,23 +2560,9 @@ private PhoenixConnection dropStatsTable(PhoenixConnection oldMetaConnection, lo } } - HBaseAdmin admin = null; - try { - admin = getAdmin(); - admin.disableTable(PhoenixDatabaseMetaData.SYSTEM_STATS_NAME_BYTES); - try { - admin.deleteTable(PhoenixDatabaseMetaData.SYSTEM_STATS_NAME_BYTES); - } catch (org.apache.hadoop.hbase.TableNotFoundException e) { - logger.debug("Stats table was not found during upgrade!!"); - } - } finally { - if (admin != null) - admin.close(); - } oldMetaConnection = metaConnection; props = PropertiesUtil.deepCopy(oldMetaConnection.getClientInfo()); - props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, - Long.toString(MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_7_0)); + props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(timestamp)); try { metaConnection = new PhoenixConnection(oldMetaConnection, ConnectionQueryServicesImpl.this, props); } finally { diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java index 199b010f94e..6cfb382c071 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java @@ -177,8 +177,8 @@ public PMetaData updateResolvedTimestamp(PTable table, long resolvedTimestamp) t @Override public PMetaData addColumn(PName tenantId, String tableName, List columns, long tableTimeStamp, - long tableSeqNum, boolean isImmutableRows, boolean isWalDisabled, boolean isMultitenant, boolean storeNulls, boolean isTransactional, long resolvedTime) throws SQLException { - return metaData = metaData.addColumn(tenantId, tableName, columns, tableTimeStamp, tableSeqNum, isImmutableRows, isWalDisabled, isMultitenant, storeNulls, isTransactional, resolvedTime); + long tableSeqNum, boolean isImmutableRows, boolean isWalDisabled, boolean isMultitenant, boolean storeNulls, boolean isTransactional, long updateCacheFrequency, long resolvedTime) throws SQLException { + return metaData = metaData.addColumn(tenantId, tableName, columns, tableTimeStamp, tableSeqNum, isImmutableRows, isWalDisabled, isMultitenant, storeNulls, isTransactional, updateCacheFrequency, resolvedTime); } @Override diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/DelegateConnectionQueryServices.java b/phoenix-core/src/main/java/org/apache/phoenix/query/DelegateConnectionQueryServices.java index 84f3e7497bc..9b721f8b265 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/query/DelegateConnectionQueryServices.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/query/DelegateConnectionQueryServices.java @@ -87,8 +87,8 @@ public PMetaData updateResolvedTimestamp(PTable table, long resolvedTimestamp) t @Override public PMetaData addColumn(PName tenantId, String tableName, List columns, long tableTimeStamp, - long tableSeqNum, boolean isImmutableRows, boolean isWalDisabled, boolean isMultitenant, boolean storeNulls, boolean isTransactional, long resolvedTime) throws SQLException { - return getDelegate().addColumn(tenantId, tableName, columns, tableTimeStamp, tableSeqNum, isImmutableRows, isWalDisabled, isMultitenant, storeNulls, isTransactional, resolvedTime); + long tableSeqNum, boolean isImmutableRows, boolean isWalDisabled, boolean isMultitenant, boolean storeNulls, boolean isTransactional, long updateCacheFrequency, long resolvedTime) throws SQLException { + return getDelegate().addColumn(tenantId, tableName, columns, tableTimeStamp, tableSeqNum, isImmutableRows, isWalDisabled, isMultitenant, storeNulls, isTransactional, updateCacheFrequency, resolvedTime); } @Override diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/MetaDataMutated.java b/phoenix-core/src/main/java/org/apache/phoenix/query/MetaDataMutated.java index 8e7a70d5a94..753b17261a1 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/query/MetaDataMutated.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/query/MetaDataMutated.java @@ -38,7 +38,7 @@ public interface MetaDataMutated { PMetaData addTable(PTable table, long resolvedTime) throws SQLException; PMetaData updateResolvedTimestamp(PTable table, long resolvedTimestamp) throws SQLException; PMetaData removeTable(PName tenantId, String tableName, String parentTableName, long tableTimeStamp) throws SQLException; - PMetaData addColumn(PName tenantId, String tableName, List columns, long tableTimeStamp, long tableSeqNum, boolean isImmutableRows, boolean isWalDisabled, boolean isMultitenant, boolean storeNulls, boolean isTransactional, long resolvedTime) throws SQLException; + PMetaData addColumn(PName tenantId, String tableName, List columns, long tableTimeStamp, long tableSeqNum, boolean isImmutableRows, boolean isWalDisabled, boolean isMultitenant, boolean storeNulls, boolean isTransactional, long updateCacheFrequency, long resolvedTime) throws SQLException; PMetaData removeColumn(PName tenantId, String tableName, List columnsToRemove, long tableTimeStamp, long tableSeqNum, long resolvedTime) throws SQLException; PMetaData addFunction(PFunction function) throws SQLException; PMetaData removeFunction(PName tenantId, String function, long functionTimeStamp) throws SQLException; diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java index c1cb0c0b79a..63d4e072b5f 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java @@ -39,9 +39,9 @@ import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.DEFAULT_VALUE; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.DISABLE_WAL; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.FUNCTION_NAME; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.GUIDE_POSTS_ROW_COUNT; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.GUIDE_POSTS_WIDTH; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.GUIDE_POST_KEY; -import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.GUIDE_POSTS_ROW_COUNT; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.IMMUTABLE_ROWS; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.INCREMENT_BY; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.INDEX_DISABLE_TIMESTAMP; @@ -96,6 +96,7 @@ import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TYPE; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TYPE_NAME; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TYPE_SEQUENCE; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.UPDATE_CACHE_FREQUENCY; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_CONSTANT; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_INDEX_ID; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_STATEMENT; @@ -257,6 +258,7 @@ public enum JoinType {INNER, LEFT_OUTER} // Column metadata (will be null for table row) IS_ROW_TIMESTAMP + " BOOLEAN, " + TRANSACTIONAL + " BOOLEAN," + + UPDATE_CACHE_FREQUENCY + " BIGINT," + "CONSTRAINT " + SYSTEM_TABLE_PK_NAME + " PRIMARY KEY (" + TENANT_ID + "," + TABLE_SCHEM + "," + TABLE_NAME + "," + COLUMN_NAME + "," + COLUMN_FAMILY + "))\n" + HConstants.VERSIONS + "=" + MetaDataProtocol.DEFAULT_MAX_META_DATA_VERSIONS + ",\n" + diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/DelegateTable.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/DelegateTable.java index 7fb90a1d1b9..e7bf961875a 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/DelegateTable.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/DelegateTable.java @@ -242,6 +242,7 @@ public boolean isTransactional() { return delegate.isTransactional(); } + @Override public int getBaseColumnCount() { return delegate.getBaseColumnCount(); } @@ -260,4 +261,9 @@ public int getRowTimestampColPos() { public String toString() { return delegate.toString(); } + + @Override + public long getUpdateCacheFrequency() { + return delegate.getUpdateCacheFrequency(); + } } diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java index ee8fba356af..ee212ed4ac1 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java @@ -76,6 +76,7 @@ import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TENANT_ID; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TRANSACTIONAL; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TYPE; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.UPDATE_CACHE_FREQUENCY; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_CONSTANT; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_INDEX_ID; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_STATEMENT; @@ -250,8 +251,9 @@ public class MetaDataClient { INDEX_TYPE + "," + STORE_NULLS + "," + BASE_COLUMN_COUNT + "," + - TRANSACTIONAL + - ") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"; + TRANSACTIONAL + "," + + UPDATE_CACHE_FREQUENCY + + ") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"; private static final String CREATE_LINK = "UPSERT INTO " + SYSTEM_CATALOG_SCHEMA + ".\"" + SYSTEM_CATALOG_TABLE + "\"( " + TENANT_ID + "," + @@ -477,8 +479,8 @@ private MetaDataMutationResult updateCache(PName tenantId, String schemaName, St // Do not make rpc to getTable if // 1. table is a system table // 2. table was already resolved as of that timestamp - if (table != null && !alwaysHitServer - && (systemTable || resolvedTimestamp == tableResolvedTimestamp)) { + if (table != null && !alwaysHitServer + && (systemTable || resolvedTimestamp == tableResolvedTimestamp || connection.getMetaDataCache().getAge(tableRef) < table.getUpdateCacheFrequency())) { return new MetaDataMutationResult(MutationCode.TABLE_ALREADY_EXISTS, QueryConstants.UNSET_TIMESTAMP, table); } @@ -1649,7 +1651,7 @@ private PTable createTableInternal(CreateTableStatement statement, byte[][] spli // Although unusual, it's possible to set a mapped VIEW as having immutable rows. // This tells Phoenix that you're managing the index maintenance yourself. if (tableType != PTableType.INDEX && (tableType != PTableType.VIEW || viewType == ViewType.MAPPED)) { - Boolean isImmutableRowsProp = (Boolean) tableProps.get(PTable.IS_IMMUTABLE_ROWS_PROP_NAME); + Boolean isImmutableRowsProp = (Boolean) TableProperty.IMMUTABLE_ROWS.getValue(tableProps); if (isImmutableRowsProp == null) { isImmutableRows = connection.getQueryServices().getProps().getBoolean(QueryServices.IMMUTABLE_ROWS_ATTRIB, QueryServicesOptions.DEFAULT_IMMUTABLE_ROWS); } else { @@ -1659,7 +1661,7 @@ private PTable createTableInternal(CreateTableStatement statement, byte[][] spli // Can't set any of these on views or shared indexes on views if (tableType != PTableType.VIEW && indexId == null) { - saltBucketNum = (Integer) tableProps.get(PhoenixDatabaseMetaData.SALT_BUCKETS); + saltBucketNum = (Integer) TableProperty.SALT_BUCKETS.getValue(tableProps); if (saltBucketNum != null) { if (saltBucketNum < 0 || saltBucketNum > SaltingUtil.MAX_BUCKET_NUM) { throw new SQLExceptionInfo.Builder(SQLExceptionCode.INVALID_BUCKET_NUM).build().buildException(); @@ -1680,16 +1682,21 @@ private PTable createTableInternal(CreateTableStatement statement, byte[][] spli if (tableType != PTableType.INDEX && (tableType != PTableType.VIEW || viewType == ViewType.MAPPED)) { Boolean multiTenantProp = (Boolean) tableProps.get(PhoenixDatabaseMetaData.MULTI_TENANT); multiTenant = Boolean.TRUE.equals(multiTenantProp); - defaultFamilyName = (String)tableProps.get(PhoenixDatabaseMetaData.DEFAULT_COLUMN_FAMILY_NAME); + defaultFamilyName = (String)TableProperty.DEFAULT_COLUMN_FAMILY.getValue(tableProps); } boolean disableWAL = false; - Boolean disableWALProp = (Boolean) tableProps.get(PhoenixDatabaseMetaData.DISABLE_WAL); + Boolean disableWALProp = (Boolean) TableProperty.DISABLE_WAL.getValue(tableProps); if (disableWALProp != null) { disableWAL = disableWALProp; } + long updateCacheFrequency = 0; + Long updateCacheFrequencyProp = (Long) TableProperty.UPDATE_CACHE_FREQUENCY.getValue(tableProps); + if (updateCacheFrequencyProp != null) { + updateCacheFrequency = updateCacheFrequencyProp; + } - Boolean storeNullsProp = (Boolean) tableProps.get(PhoenixDatabaseMetaData.STORE_NULLS); + Boolean storeNullsProp = (Boolean) TableProperty.STORE_NULLS.getValue(tableProps); if (storeNullsProp == null) { if (parent == null) { storeNulls = connection.getQueryServices().getProps().getBoolean( @@ -1700,7 +1707,7 @@ private PTable createTableInternal(CreateTableStatement statement, byte[][] spli } else { storeNulls = storeNullsProp; } - Boolean transactionalProp = (Boolean) tableProps.get(PhoenixDatabaseMetaData.TRANSACTIONAL); + Boolean transactionalProp = (Boolean) TableProperty.TRANSACTIONAL.getValue(tableProps); if (transactionalProp != null && parent != null) { throw new SQLExceptionInfo.Builder(SQLExceptionCode.ONLY_TABLE_MAY_BE_DECLARED_TRANSACTIONAL) .setSchemaName(schemaName).setTableName(tableName) @@ -1731,7 +1738,8 @@ private PTable createTableInternal(CreateTableStatement statement, byte[][] spli .build().buildException(); } - + // Put potentially inferred value into tableProps as it's used by the createTable call below + // to determine which coprocessors to install on the new table. tableProps.put(PhoenixDatabaseMetaData.TRANSACTIONAL, transactional); if (transactional) { // If TTL set, use Tephra TTL property name instead @@ -2020,7 +2028,7 @@ private PTable createTableInternal(CreateTableStatement statement, byte[][] spli Collections.emptyList(), isImmutableRows, Collections.emptyList(), defaultFamilyName == null ? null : PNameFactory.newName(defaultFamilyName), null, - Boolean.TRUE.equals(disableWAL), false, false, null, indexId, indexType, true, false); + Boolean.TRUE.equals(disableWAL), false, false, null, indexId, indexType, true, false, 0); connection.addTable(table, MetaDataProtocol.MIN_TABLE_TIMESTAMP); } else if (tableType == PTableType.INDEX && indexId == null) { if (tableProps.get(HTableDescriptor.MAX_FILESIZE) == null) { @@ -2123,6 +2131,7 @@ public boolean isViewReferenced() { tableUpsert.setInt(20, BASE_TABLE_BASE_COLUMN_COUNT); } tableUpsert.setBoolean(21, transactional); + tableUpsert.setLong(22, updateCacheFrequency); tableUpsert.execute(); if (asyncCreatedDate != null) { @@ -2185,7 +2194,7 @@ public boolean isViewReferenced() { PTable.INITIAL_SEQ_NUM, pkName == null ? null : PNameFactory.newName(pkName), saltBucketNum, columns, dataTableName == null ? null : newSchemaName, dataTableName == null ? null : PNameFactory.newName(dataTableName), Collections.emptyList(), isImmutableRows, physicalNames, defaultFamilyName == null ? null : PNameFactory.newName(defaultFamilyName), viewStatement, Boolean.TRUE.equals(disableWAL), multiTenant, storeNulls, viewType, - indexId, indexType, rowKeyOrderOptimizable, transactional); + indexId, indexType, rowKeyOrderOptimizable, transactional, updateCacheFrequency); result = new MetaDataMutationResult(code, result.getMutationTime(), table, true); addTableToCache(result); return table; @@ -2508,12 +2517,12 @@ private MutationCode processMutationResult(String schemaName, String tableName, return mutationCode; } - private long incrementTableSeqNum(PTable table, PTableType expectedType, int columnCountDelta, Boolean isTransactional) throws SQLException { - return incrementTableSeqNum(table, expectedType, columnCountDelta, isTransactional, null, null, null, null); + private long incrementTableSeqNum(PTable table, PTableType expectedType, int columnCountDelta, Boolean isTransactional, Long updateCacheFrequency) throws SQLException { + return incrementTableSeqNum(table, expectedType, columnCountDelta, isTransactional, updateCacheFrequency, null, null, null, null); } private long incrementTableSeqNum(PTable table, PTableType expectedType, int columnCountDelta, - Boolean isTransactional, Boolean isImmutableRows, Boolean disableWAL, Boolean isMultiTenant, Boolean storeNulls) + Boolean isTransactional, Long updateCacheFrequency, Boolean isImmutableRows, Boolean disableWAL, Boolean isMultiTenant, Boolean storeNulls) throws SQLException { String schemaName = table.getSchemaName().getString(); String tableName = table.getTableName().getString(); @@ -2548,6 +2557,9 @@ private long incrementTableSeqNum(PTable table, PTableType expectedType, int col if (isTransactional != null) { mutateBooleanProperty(tenantId, schemaName, tableName, TRANSACTIONAL, isTransactional); } + if (updateCacheFrequency != null) { + mutateLongProperty(tenantId, schemaName, tableName, UPDATE_CACHE_FREQUENCY, updateCacheFrequency); + } return seqNum; } @@ -2567,6 +2579,22 @@ private void mutateBooleanProperty(String tenantId, String schemaName, String ta tableBoolUpsert.execute(); } + private void mutateLongProperty(String tenantId, String schemaName, String tableName, + String propertyName, long propertyValue) throws SQLException { + String updatePropertySql = "UPSERT INTO " + SYSTEM_CATALOG_SCHEMA + ".\"" + SYSTEM_CATALOG_TABLE + "\"( " + + TENANT_ID + "," + + TABLE_SCHEM + "," + + TABLE_NAME + "," + + propertyName + + ") VALUES (?, ?, ?, ?)"; + PreparedStatement tableBoolUpsert = connection.prepareStatement(updatePropertySql); + tableBoolUpsert.setString(1, tenantId); + tableBoolUpsert.setString(2, schemaName); + tableBoolUpsert.setString(3, tableName); + tableBoolUpsert.setLong(4, propertyValue); + tableBoolUpsert.execute(); + } + public MutationState addColumn(AddColumnStatement statement) throws SQLException { connection.rollback(); boolean wasAutoCommit = connection.getAutoCommit(); @@ -2582,6 +2610,7 @@ public MutationState addColumn(AddColumnStatement statement) throws SQLException Boolean disableWALProp = null; Boolean storeNullsProp = null; Boolean isTransactionalProp = null; + Long updateCacheFrequencyProp = null; ListMultimap> stmtProperties = statement.getProps(); Map>> properties = new HashMap<>(stmtProperties.size()); @@ -2596,17 +2625,21 @@ public MutationState addColumn(AddColumnStatement statement) throws SQLException for (Pair prop : propsList) { String propName = prop.getFirst(); if (TableProperty.isPhoenixTableProperty(propName)) { - TableProperty.valueOf(propName).validate(true, !family.equals(QueryConstants.ALL_FAMILY_PROPERTIES_KEY), table.getType()); + TableProperty tableProp = TableProperty.valueOf(propName); + tableProp.validate(true, !family.equals(QueryConstants.ALL_FAMILY_PROPERTIES_KEY), table.getType()); + Object value = tableProp.getValue(prop.getSecond()); if (propName.equals(PTable.IS_IMMUTABLE_ROWS_PROP_NAME)) { - isImmutableRowsProp = (Boolean)prop.getSecond(); + isImmutableRowsProp = (Boolean)value; } else if (propName.equals(PhoenixDatabaseMetaData.MULTI_TENANT)) { - multiTenantProp = (Boolean)prop.getSecond(); + multiTenantProp = (Boolean)value; } else if (propName.equals(DISABLE_WAL)) { - disableWALProp = (Boolean)prop.getSecond(); + disableWALProp = (Boolean)value; } else if (propName.equals(STORE_NULLS)) { - storeNullsProp = (Boolean)prop.getSecond(); + storeNullsProp = (Boolean)value; } else if (propName.equals(TRANSACTIONAL)) { - isTransactionalProp = (Boolean)prop.getSecond(); + isTransactionalProp = (Boolean)value; + } else if (propName.equals(UPDATE_CACHE_FREQUENCY)) { + updateCacheFrequencyProp = (Long)value; } } } @@ -2662,6 +2695,13 @@ public MutationState addColumn(AddColumnStatement statement) throws SQLException changingPhoenixTableProperty = true; } } + Long updateCacheFrequency = null; + if (updateCacheFrequencyProp != null) { + if (updateCacheFrequencyProp.longValue() != table.getUpdateCacheFrequency()) { + updateCacheFrequency = updateCacheFrequencyProp; + changingPhoenixTableProperty = true; + } + } Boolean storeNulls = null; if (storeNullsProp != null) { if (storeNullsProp.booleanValue() != table.getStoreNulls()) { @@ -2795,14 +2835,14 @@ public MutationState addColumn(AddColumnStatement statement) throws SQLException if (!table.getIndexes().isEmpty() && (numPkColumnsAdded>0 || nonTxToTx)) { for (PTable index : table.getIndexes()) { - incrementTableSeqNum(index, index.getType(), numPkColumnsAdded, nonTxToTx ? Boolean.TRUE : null); + incrementTableSeqNum(index, index.getType(), numPkColumnsAdded, nonTxToTx ? Boolean.TRUE : null, updateCacheFrequency); } tableMetaData.addAll(connection.getMutationState().toMutations(timeStamp).next().getSecond()); connection.rollback(); } long seqNum = table.getSequenceNumber(); if (changingPhoenixTableProperty || columnDefs.size() > 0) { - seqNum = incrementTableSeqNum(table, statement.getTableType(), columnDefs.size(), isTransactional, isImmutableRows, disableWAL, multiTenant, storeNulls); + seqNum = incrementTableSeqNum(table, statement.getTableType(), columnDefs.size(), isTransactional, updateCacheFrequency, isImmutableRows, disableWAL, multiTenant, storeNulls); tableMetaData.addAll(connection.getMutationState().toMutations(timeStamp).next().getSecond()); connection.rollback(); } @@ -2844,19 +2884,26 @@ public MutationState addColumn(AddColumnStatement statement) throws SQLException // Only update client side cache if we aren't adding a PK column to a table with indexes or // transitioning a table from non transactional to transactional. // We could update the cache manually then too, it'd just be a pain. + String fullTableName = SchemaUtil.getTableName(schemaName, tableName); + long resolvedTimeStamp = TransactionUtil.getResolvedTime(connection, result); if (table.getIndexes().isEmpty() || (numPkColumnsAdded==0 && !nonTxToTx)) { - connection.addColumn( - tenantId, - SchemaUtil.getTableName(schemaName, tableName), - columns, - result.getMutationTime(), - seqNum, - isImmutableRows == null ? table.isImmutableRows() : isImmutableRows, - disableWAL == null ? table.isWALDisabled() : disableWAL, - multiTenant == null ? table.isMultiTenant() : multiTenant, - storeNulls == null ? table.getStoreNulls() : storeNulls, - isTransactional == null ? table.isTransactional() : isTransactional, - TransactionUtil.getResolvedTime(connection, result)); + connection.addColumn( + tenantId, + fullTableName, + columns, + result.getMutationTime(), + seqNum, + isImmutableRows == null ? table.isImmutableRows() : isImmutableRows, + disableWAL == null ? table.isWALDisabled() : disableWAL, + multiTenant == null ? table.isMultiTenant() : multiTenant, + storeNulls == null ? table.getStoreNulls() : storeNulls, + isTransactional == null ? table.isTransactional() : isTransactional, + updateCacheFrequency == null ? table.getUpdateCacheFrequency() : updateCacheFrequency, + resolvedTimeStamp); + } else if (updateCacheFrequency != null) { + // Force removal from cache as the update cache frequency has changed + // Note that clients outside this JVM won't be affected. + connection.removeTable(tenantId, fullTableName, null, resolvedTimeStamp); } // Delete rows in view index if we haven't dropped it already // We only need to do this if the multiTenant transitioned to false @@ -3036,7 +3083,7 @@ else if (coveredColumns.contains(columnToDropRef)) { } } if(!indexColumnsToDrop.isEmpty()) { - incrementTableSeqNum(index, index.getType(), -indexColumnsToDrop.size(), null); + incrementTableSeqNum(index, index.getType(), -indexColumnsToDrop.size(), null, null); dropColumnMutations(index, indexColumnsToDrop, tableMetaData); } @@ -3045,7 +3092,7 @@ else if (coveredColumns.contains(columnToDropRef)) { tableMetaData.addAll(connection.getMutationState().toMutations(timeStamp).next().getSecond()); connection.rollback(); - long seqNum = incrementTableSeqNum(table, statement.getTableType(), -tableColumnsToDrop.size(), null); + long seqNum = incrementTableSeqNum(table, statement.getTableType(), -tableColumnsToDrop.size(), null, null); tableMetaData.addAll(connection.getMutationState().toMutations(timeStamp).next().getSecond()); connection.rollback(); // Force table header to be first in list diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/PMetaData.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/PMetaData.java index a3103bff9a4..3adcb7e7aff 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/PMetaData.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/PMetaData.java @@ -17,8 +17,6 @@ */ package org.apache.phoenix.schema; -import java.sql.SQLException; - import org.apache.phoenix.parse.PFunction; import org.apache.phoenix.query.MetaDataMutated; @@ -34,4 +32,5 @@ public static interface Pruner { public PMetaData pruneTables(Pruner pruner); public PFunction getFunction(PTableKey key) throws FunctionNotFoundException; public PMetaData pruneFunctions(Pruner pruner); + public long getAge(PTableRef ref); } diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/PMetaDataImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/PMetaDataImpl.java index 9e4460d9a08..66b4af3a8d1 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/PMetaDataImpl.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/PMetaDataImpl.java @@ -170,6 +170,10 @@ public PTable putDuplicate(PTableKey key, PTable value, long resolvedTime) { return put(key, new PTableRef(value, timeKeeper.getCurrentTime(), 0, resolvedTime)); } + public long getAge(PTableRef ref) { + return timeKeeper.getCurrentTime() - ref.getCreateTime(); + } + public PTable remove(PTableKey key) { PTableRef value = this.tables.remove(key); if (value == null) { @@ -313,7 +317,7 @@ public PMetaData addTable(PTable table, long resolvedTime) throws SQLException { } @Override - public PMetaData addColumn(PName tenantId, String tableName, List columnsToAdd, long tableTimeStamp, long tableSeqNum, boolean isImmutableRows, boolean isWalDisabled, boolean isMultitenant, boolean storeNulls, boolean isTransactional, long resolvedTime) throws SQLException { + public PMetaData addColumn(PName tenantId, String tableName, List columnsToAdd, long tableTimeStamp, long tableSeqNum, boolean isImmutableRows, boolean isWalDisabled, boolean isMultitenant, boolean storeNulls, boolean isTransactional, long updateCacheFrequency, long resolvedTime) throws SQLException { PTableRef oldTableRef = metaData.get(new PTableKey(tenantId, tableName)); if (oldTableRef == null) { return this; @@ -327,7 +331,9 @@ public PMetaData addColumn(PName tenantId, String tableName, List colum newColumns.addAll(oldColumns); newColumns.addAll(columnsToAdd); } - PTable newTable = PTableImpl.makePTable(oldTableRef.getTable(), tableTimeStamp, tableSeqNum, newColumns, isImmutableRows, isWalDisabled, isMultitenant, storeNulls, isTransactional); + PTable newTable = PTableImpl.makePTable(oldTableRef.getTable(), + tableTimeStamp, tableSeqNum, newColumns, isImmutableRows, + isWalDisabled, isMultitenant, storeNulls, isTransactional, updateCacheFrequency); return addTable(newTable, resolvedTime); } @@ -472,4 +478,9 @@ public PMetaData pruneFunctions(Pruner pruner) { return new PMetaDataImpl(clone); } + + @Override + public long getAge(PTableRef ref) { + return this.metaData.getAge(ref); + } } diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/PTable.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/PTable.java index ec97394a2d5..4a338f6bc65 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/PTable.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/PTable.java @@ -339,4 +339,5 @@ public static LinkType fromSerializedValue(byte serializedValue) { * -1 if there is no such column. */ int getRowTimestampColPos(); + long getUpdateCacheFrequency(); } diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java index 1805d94bffd..1611466451f 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java @@ -135,6 +135,7 @@ public class PTableImpl implements PTable { private boolean rowKeyOrderOptimizable; // TODO: remove when required that tables have been upgrade for PHOENIX-2067 private boolean hasColumnsRequiringUpgrade; // TODO: remove when required that tables have been upgrade for PHOENIX-2067 private int rowTimestampColPos; + private long updateCacheFrequency; public PTableImpl() { this.indexes = Collections.emptyList(); @@ -205,7 +206,7 @@ public static PTableImpl makePTable(PTable table, long timeStamp, List i table.getSequenceNumber(), table.getPKName(), table.getBucketNum(), getColumnsToClone(table), parentSchemaName, table.getParentTableName(), indexes, table.isImmutableRows(), table.getPhysicalNames(), table.getDefaultFamilyName(), viewStatement, table.isWALDisabled(), table.isMultiTenant(), table.getStoreNulls(), table.getViewType(), table.getViewIndexId(), table.getIndexType(), - table.getTableStats(), table.getBaseColumnCount(), table.rowKeyOrderOptimizable(), table.isTransactional()); + table.getTableStats(), table.getBaseColumnCount(), table.rowKeyOrderOptimizable(), table.isTransactional(), table.getUpdateCacheFrequency()); } public static PTableImpl makePTable(PTable table, List columns) throws SQLException { @@ -214,7 +215,7 @@ public static PTableImpl makePTable(PTable table, List columns) throws table.getSequenceNumber(), table.getPKName(), table.getBucketNum(), columns, table.getParentSchemaName(), table.getParentTableName(), table.getIndexes(), table.isImmutableRows(), table.getPhysicalNames(), table.getDefaultFamilyName(), table.getViewStatement(), table.isWALDisabled(), table.isMultiTenant(), table.getStoreNulls(), table.getViewType(), table.getViewIndexId(), table.getIndexType(), - table.getTableStats(), table.getBaseColumnCount(), table.rowKeyOrderOptimizable(), table.isTransactional()); + table.getTableStats(), table.getBaseColumnCount(), table.rowKeyOrderOptimizable(), table.isTransactional(), table.getUpdateCacheFrequency()); } public static PTableImpl makePTable(PTable table, long timeStamp, long sequenceNumber, List columns) throws SQLException { @@ -223,7 +224,7 @@ public static PTableImpl makePTable(PTable table, long timeStamp, long sequenceN sequenceNumber, table.getPKName(), table.getBucketNum(), columns, table.getParentSchemaName(), table.getParentTableName(), table.getIndexes(), table.isImmutableRows(), table.getPhysicalNames(), table.getDefaultFamilyName(), table.getViewStatement(), table.isWALDisabled(), table.isMultiTenant(), table.getStoreNulls(), table.getViewType(), table.getViewIndexId(), table.getIndexType(), table.getTableStats(), - table.getBaseColumnCount(), table.rowKeyOrderOptimizable(), table.isTransactional()); + table.getBaseColumnCount(), table.rowKeyOrderOptimizable(), table.isTransactional(), table.getUpdateCacheFrequency()); } public static PTableImpl makePTable(PTable table, long timeStamp, long sequenceNumber, List columns, boolean isImmutableRows) throws SQLException { @@ -232,16 +233,17 @@ public static PTableImpl makePTable(PTable table, long timeStamp, long sequenceN sequenceNumber, table.getPKName(), table.getBucketNum(), columns, table.getParentSchemaName(), table.getParentTableName(), table.getIndexes(), isImmutableRows, table.getPhysicalNames(), table.getDefaultFamilyName(), table.getViewStatement(), table.isWALDisabled(), table.isMultiTenant(), table.getStoreNulls(), table.getViewType(), table.getViewIndexId(), - table.getIndexType(), table.getTableStats(), table.getBaseColumnCount(), table.rowKeyOrderOptimizable(), table.isTransactional()); + table.getIndexType(), table.getTableStats(), table.getBaseColumnCount(), table.rowKeyOrderOptimizable(), table.isTransactional(), table.getUpdateCacheFrequency()); } - public static PTableImpl makePTable(PTable table, long timeStamp, long sequenceNumber, List columns, boolean isImmutableRows, boolean isWalDisabled, boolean isMultitenant, boolean storeNulls, boolean isTransactional) throws SQLException { + public static PTableImpl makePTable(PTable table, long timeStamp, long sequenceNumber, List columns, boolean isImmutableRows, boolean isWalDisabled, + boolean isMultitenant, boolean storeNulls, boolean isTransactional, long updateCacheFrequency) throws SQLException { return new PTableImpl( table.getTenantId(), table.getSchemaName(), table.getTableName(), table.getType(), table.getIndexState(), timeStamp, sequenceNumber, table.getPKName(), table.getBucketNum(), columns, table.getParentSchemaName(), table.getParentTableName(), table.getIndexes(), isImmutableRows, table.getPhysicalNames(), table.getDefaultFamilyName(), table.getViewStatement(), isWalDisabled, isMultitenant, storeNulls, table.getViewType(), table.getViewIndexId(), table.getIndexType(), table.getTableStats(), - table.getBaseColumnCount(), table.rowKeyOrderOptimizable(), isTransactional); + table.getBaseColumnCount(), table.rowKeyOrderOptimizable(), isTransactional, updateCacheFrequency); } public static PTableImpl makePTable(PTable table, PIndexState state) throws SQLException { @@ -251,7 +253,7 @@ public static PTableImpl makePTable(PTable table, PIndexState state) throws SQLE table.getParentSchemaName(), table.getParentTableName(), table.getIndexes(), table.isImmutableRows(), table.getPhysicalNames(), table.getDefaultFamilyName(), table.getViewStatement(), table.isWALDisabled(), table.isMultiTenant(), table.getStoreNulls(), table.getViewType(), table.getViewIndexId(), table.getIndexType(), - table.getTableStats(), table.getBaseColumnCount(), table.rowKeyOrderOptimizable(), table.isTransactional()); + table.getTableStats(), table.getBaseColumnCount(), table.rowKeyOrderOptimizable(), table.isTransactional(), table.getUpdateCacheFrequency()); } public static PTableImpl makePTable(PTable table, boolean rowKeyOrderOptimizable) throws SQLException { @@ -261,7 +263,7 @@ public static PTableImpl makePTable(PTable table, boolean rowKeyOrderOptimizable table.getParentSchemaName(), table.getParentTableName(), table.getIndexes(), table.isImmutableRows(), table.getPhysicalNames(), table.getDefaultFamilyName(), table.getViewStatement(), table.isWALDisabled(), table.isMultiTenant(), table.getStoreNulls(), table.getViewType(), table.getViewIndexId(), table.getIndexType(), table.getTableStats(), - table.getBaseColumnCount(), rowKeyOrderOptimizable, table.isTransactional()); + table.getBaseColumnCount(), rowKeyOrderOptimizable, table.isTransactional(), table.getUpdateCacheFrequency()); } public static PTableImpl makePTable(PTable table, PTableStats stats) throws SQLException { @@ -271,28 +273,33 @@ public static PTableImpl makePTable(PTable table, PTableStats stats) throws SQLE table.getParentSchemaName(), table.getParentTableName(), table.getIndexes(), table.isImmutableRows(), table.getPhysicalNames(), table.getDefaultFamilyName(), table.getViewStatement(), table.isWALDisabled(), table.isMultiTenant(), table.getStoreNulls(), table.getViewType(), table.getViewIndexId(), table.getIndexType(), stats, - table.getBaseColumnCount(), table.rowKeyOrderOptimizable(), table.isTransactional()); + table.getBaseColumnCount(), table.rowKeyOrderOptimizable(), table.isTransactional(), table.getUpdateCacheFrequency()); } - public static PTableImpl makePTable(PName tenantId, PName schemaName, PName tableName, PTableType type, PIndexState state, long timeStamp, long sequenceNumber, - PName pkName, Integer bucketNum, List columns, PName dataSchemaName, PName dataTableName, List indexes, - boolean isImmutableRows, List physicalNames, PName defaultFamilyName, String viewExpression, boolean disableWAL, boolean multiTenant, - boolean storeNulls, ViewType viewType, Short viewIndexId, IndexType indexType, boolean rowKeyOrderOptimizable, boolean isTransactional) throws SQLException { + public static PTableImpl makePTable(PName tenantId, PName schemaName, PName tableName, PTableType type, + PIndexState state, long timeStamp, long sequenceNumber, PName pkName, Integer bucketNum, + List columns, PName dataSchemaName, PName dataTableName, List indexes, + boolean isImmutableRows, List physicalNames, PName defaultFamilyName, String viewExpression, + boolean disableWAL, boolean multiTenant, boolean storeNulls, ViewType viewType, Short viewIndexId, + IndexType indexType, boolean rowKeyOrderOptimizable, boolean isTransactional, long updateCacheFrequency) throws SQLException { return new PTableImpl(tenantId, schemaName, tableName, type, state, timeStamp, sequenceNumber, pkName, bucketNum, columns, dataSchemaName, dataTableName, indexes, isImmutableRows, physicalNames, defaultFamilyName, viewExpression, disableWAL, multiTenant, storeNulls, viewType, viewIndexId, - indexType, PTableStats.EMPTY_STATS, QueryConstants.BASE_TABLE_BASE_COLUMN_COUNT, rowKeyOrderOptimizable, isTransactional); + indexType, PTableStats.EMPTY_STATS, QueryConstants.BASE_TABLE_BASE_COLUMN_COUNT, rowKeyOrderOptimizable, isTransactional, updateCacheFrequency); } public static PTableImpl makePTable(PName tenantId, PName schemaName, PName tableName, PTableType type, PIndexState state, long timeStamp, long sequenceNumber, PName pkName, Integer bucketNum, List columns, PName dataSchemaName, PName dataTableName, List indexes, boolean isImmutableRows, List physicalNames, PName defaultFamilyName, String viewExpression, - boolean disableWAL, boolean multiTenant, boolean storeNulls, ViewType viewType, Short viewIndexId, IndexType indexType, @NotNull PTableStats stats, int baseColumnCount, boolean rowKeyOrderOptimizable, boolean isTransactional) + boolean disableWAL, boolean multiTenant, boolean storeNulls, ViewType viewType, Short viewIndexId, + IndexType indexType, boolean rowKeyOrderOptimizable, boolean isTransactional, long updateCacheFrequency, + @NotNull PTableStats stats, int baseColumnCount) throws SQLException { return new PTableImpl(tenantId, schemaName, tableName, type, state, timeStamp, sequenceNumber, pkName, bucketNum, columns, dataSchemaName, dataTableName, indexes, isImmutableRows, physicalNames, - defaultFamilyName, viewExpression, disableWAL, multiTenant, storeNulls, viewType, viewIndexId, indexType, stats, baseColumnCount, rowKeyOrderOptimizable, isTransactional); + defaultFamilyName, viewExpression, disableWAL, multiTenant, storeNulls, viewType, viewIndexId, + indexType, stats, baseColumnCount, rowKeyOrderOptimizable, isTransactional, updateCacheFrequency); } private PTableImpl(PName tenantId, PName schemaName, PName tableName, PTableType type, PIndexState state, @@ -300,12 +307,18 @@ private PTableImpl(PName tenantId, PName schemaName, PName tableName, PTableType PName parentSchemaName, PName parentTableName, List indexes, boolean isImmutableRows, List physicalNames, PName defaultFamilyName, String viewExpression, boolean disableWAL, boolean multiTenant, boolean storeNulls, ViewType viewType, Short viewIndexId, IndexType indexType, - PTableStats stats, int baseColumnCount, boolean rowKeyOrderOptimizable, boolean isTransactional) throws SQLException { + PTableStats stats, int baseColumnCount, boolean rowKeyOrderOptimizable, boolean isTransactional, long updateCacheFrequency) throws SQLException { init(tenantId, schemaName, tableName, type, state, timeStamp, sequenceNumber, pkName, bucketNum, columns, stats, schemaName, parentTableName, indexes, isImmutableRows, physicalNames, defaultFamilyName, - viewExpression, disableWAL, multiTenant, storeNulls, viewType, viewIndexId, indexType, baseColumnCount, rowKeyOrderOptimizable, isTransactional); + viewExpression, disableWAL, multiTenant, storeNulls, viewType, viewIndexId, indexType, baseColumnCount, rowKeyOrderOptimizable, + isTransactional, updateCacheFrequency); } + @Override + public long getUpdateCacheFrequency() { + return updateCacheFrequency; + } + @Override public boolean isMultiTenant() { return multiTenant; @@ -331,7 +344,7 @@ private void init(PName tenantId, PName schemaName, PName tableName, PTableType PName pkName, Integer bucketNum, List columns, PTableStats stats, PName parentSchemaName, PName parentTableName, List indexes, boolean isImmutableRows, List physicalNames, PName defaultFamilyName, String viewExpression, boolean disableWAL, boolean multiTenant, boolean storeNulls, ViewType viewType, Short viewIndexId, - IndexType indexType , int baseColumnCount, boolean rowKeyOrderOptimizable, boolean isTransactional) throws SQLException { + IndexType indexType , int baseColumnCount, boolean rowKeyOrderOptimizable, boolean isTransactional, long updateCacheFrequency) throws SQLException { Preconditions.checkNotNull(schemaName); Preconditions.checkArgument(tenantId==null || tenantId.getBytes().length > 0); // tenantId should be null or not empty int estimatedSize = SizedUtil.OBJECT_SIZE * 2 + 23 * SizedUtil.POINTER_SIZE + 4 * SizedUtil.INT_SIZE + 2 * SizedUtil.LONG_SIZE + 2 * SizedUtil.INT_OBJECT_SIZE + @@ -363,6 +376,7 @@ private void init(PName tenantId, PName schemaName, PName tableName, PTableType this.isTransactional = isTransactional; this.tableStats = stats; this.rowKeyOrderOptimizable = rowKeyOrderOptimizable; + this.updateCacheFrequency = updateCacheFrequency; List pkColumns; PColumn[] allColumns; @@ -1060,12 +1074,18 @@ public static PTable createFromProto(PTableProtos.PTable table) { if (table.hasRowKeyOrderOptimizable()) { rowKeyOrderOptimizable = table.getRowKeyOrderOptimizable(); } + long updateCacheFrequency = 0; + if (table.hasUpdateCacheFrequency()) { + updateCacheFrequency = table.getUpdateCacheFrequency(); + } + try { PTableImpl result = new PTableImpl(); result.init(tenantId, schemaName, tableName, tableType, indexState, timeStamp, sequenceNumber, pkName, - (bucketNum == NO_SALTING) ? null : bucketNum, columns, stats, schemaName,dataTableName, indexes, - isImmutableRows, physicalNames, defaultFamilyName, viewStatement, disableWAL, - multiTenant, storeNulls, viewType, viewIndexId, indexType, baseColumnCount, rowKeyOrderOptimizable, isTransactional); + (bucketNum == NO_SALTING) ? null : bucketNum, columns, stats, schemaName,dataTableName, indexes, + isImmutableRows, physicalNames, defaultFamilyName, viewStatement, disableWAL, + multiTenant, storeNulls, viewType, viewIndexId, indexType, baseColumnCount, rowKeyOrderOptimizable, + isTransactional, updateCacheFrequency); return result; } catch (SQLException e) { throw new RuntimeException(e); // Impossible @@ -1157,6 +1177,7 @@ public static PTableProtos.PTable toProto(PTable table) { } builder.setBaseColumnCount(table.getBaseColumnCount()); builder.setRowKeyOrderOptimizable(table.rowKeyOrderOptimizable()); + builder.setUpdateCacheFrequency(table.getUpdateCacheFrequency()); return builder.build(); } diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableRef.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableRef.java index 83d0b42f82f..c4bc5107525 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableRef.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableRef.java @@ -20,15 +20,16 @@ public class PTableRef { private final PTable table; private final int estSize; + private final long createTime; + private final long resolvedTimeStamp; private volatile long lastAccessTime; - // timestamp (scn or txn timestamp at which rpc to fetch the table was made) - private long resolvedTimeStamp; public PTableRef(PTable table, long lastAccessTime, int estSize, long resolvedTime) { this.table = table; this.lastAccessTime = lastAccessTime; this.estSize = estSize; this.resolvedTimeStamp = resolvedTime; + this.createTime = lastAccessTime; } public PTableRef(PTable table, long lastAccessTime, long resolvedTime) { @@ -39,6 +40,14 @@ public PTableRef(PTableRef tableRef) { this (tableRef.table, tableRef.lastAccessTime, tableRef.estSize, tableRef.resolvedTimeStamp); } + /** + * Tracks how long this entry has been in the cache + * @return time in milliseconds for how long this entry has been in the cache. + */ + public long getCreateTime() { + return createTime; + } + public PTable getTable() { return table; } @@ -58,8 +67,4 @@ public long getLastAccessTime() { public void setLastAccessTime(long lastAccessTime) { this.lastAccessTime = lastAccessTime; } - - public void setResolvedTimeStamp(long resolvedTimeStamp) { - this.resolvedTimeStamp = resolvedTimeStamp; - } } \ No newline at end of file diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/TableProperty.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/TableProperty.java index 091b92961eb..5967d8b54b9 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/TableProperty.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/TableProperty.java @@ -26,6 +26,7 @@ import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.DEFAULT_COLUMN_FAMILY_NAME; import java.sql.SQLException; +import java.util.Map; import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.phoenix.exception.SQLExceptionCode; @@ -49,6 +50,23 @@ public enum TableProperty { STORE_NULLS(PhoenixDatabaseMetaData.STORE_NULLS, COLUMN_FAMILY_NOT_ALLOWED_TABLE_PROPERTY, true, false), TRANSACTIONAL(PhoenixDatabaseMetaData.TRANSACTIONAL, COLUMN_FAMILY_NOT_ALLOWED_TABLE_PROPERTY, true, false), + + UPDATE_CACHE_FREQUENCY(PhoenixDatabaseMetaData.UPDATE_CACHE_FREQUENCY, true, true) { + @Override + public Object getValue(Object value) { + if (value instanceof String) { + String strValue = (String) value; + if ("ALWAYS".equalsIgnoreCase(strValue)) { + return 0L; + } else if ("NEVER".equalsIgnoreCase(strValue)) { + return Long.MAX_VALUE; + } + } else { + return value == null ? null : ((Number) value).longValue(); + } + return value; + } + }, ; @@ -87,6 +105,14 @@ public static boolean isPhoenixTableProperty(String property) { return true; } + public Object getValue(Object value) { + return value; + } + + public Object getValue(Map props) { + return getValue(props.get(this.toString())); + } + // isQualified is true if column family name is specified in property name public void validate(boolean isMutating, boolean isQualified, PTableType tableType) throws SQLException { checkForColumnFamily(isQualified); diff --git a/phoenix-core/src/test/java/org/apache/phoenix/execute/CorrelatePlanTest.java b/phoenix-core/src/test/java/org/apache/phoenix/execute/CorrelatePlanTest.java index 9d5e17ce63f..72f3e015c4d 100644 --- a/phoenix-core/src/test/java/org/apache/phoenix/execute/CorrelatePlanTest.java +++ b/phoenix-core/src/test/java/org/apache/phoenix/execute/CorrelatePlanTest.java @@ -36,12 +36,12 @@ import org.apache.phoenix.compile.ColumnResolver; import org.apache.phoenix.compile.FromCompiler; import org.apache.phoenix.compile.JoinCompiler; +import org.apache.phoenix.compile.OrderByCompiler.OrderBy; import org.apache.phoenix.compile.QueryPlan; import org.apache.phoenix.compile.RowProjector; import org.apache.phoenix.compile.SequenceManager; import org.apache.phoenix.compile.StatementContext; import org.apache.phoenix.compile.TupleProjectionCompiler; -import org.apache.phoenix.compile.OrderByCompiler.OrderBy; import org.apache.phoenix.coprocessor.MetaDataProtocol; import org.apache.phoenix.exception.SQLExceptionCode; import org.apache.phoenix.expression.ComparisonExpression; @@ -233,7 +233,7 @@ private TableRef createProjectedTableFromLiterals(Object[] row) { PTableType.SUBQUERY, null, MetaDataProtocol.MIN_TABLE_TIMESTAMP, PTable.INITIAL_SEQ_NUM, null, null, columns, null, null, Collections.emptyList(), false, Collections.emptyList(), null, null, false, false, false, null, - null, null, true, false); + null, null, true, false, 0); TableRef sourceTable = new TableRef(pTable); List sourceColumnRefs = Lists. newArrayList(); for (PColumn column : sourceTable.getTable().getColumns()) { diff --git a/phoenix-core/src/test/java/org/apache/phoenix/schema/PMetaDataImplTest.java b/phoenix-core/src/test/java/org/apache/phoenix/schema/PMetaDataImplTest.java index 14b5f1f4218..9c926797a27 100644 --- a/phoenix-core/src/test/java/org/apache/phoenix/schema/PMetaDataImplTest.java +++ b/phoenix-core/src/test/java/org/apache/phoenix/schema/PMetaDataImplTest.java @@ -17,29 +17,37 @@ */ package org.apache.phoenix.schema; -import com.google.common.collect.Sets; -import org.apache.hadoop.hbase.HConstants; -import org.apache.phoenix.util.TimeKeeper; -import org.junit.Test; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; import java.sql.SQLException; import java.util.Set; -import static org.junit.Assert.assertEquals; +import org.apache.hadoop.hbase.HConstants; +import org.apache.phoenix.util.TimeKeeper; +import org.junit.Test; + +import com.google.common.collect.Sets; public class PMetaDataImplTest { - private static PMetaData addToTable(PMetaData metaData, String name, int size) throws SQLException { + private static PMetaData addToTable(PMetaData metaData, String name, int size, TestTimeKeeper timeKeeper) throws SQLException { PTable table = new PSizedTable(new PTableKey(null,name), size); - return metaData.addTable(table, System.currentTimeMillis()); + PMetaData newMetaData = metaData.addTable(table, System.currentTimeMillis()); + timeKeeper.incrementTime(); + return newMetaData; } - private static PMetaData removeFromTable(PMetaData metaData, String name) throws SQLException { - return metaData.removeTable(null, name, null, HConstants.LATEST_TIMESTAMP); + private static PMetaData removeFromTable(PMetaData metaData, String name, TestTimeKeeper timeKeeper) throws SQLException { + PMetaData newMetaData = metaData.removeTable(null, name, null, HConstants.LATEST_TIMESTAMP); + timeKeeper.incrementTime(); + return newMetaData; } - private static PTable getFromTable(PMetaData metaData, String name) throws TableNotFoundException { - return metaData.getTableRef(new PTableKey(null,name)).getTable(); + private static PTable getFromTable(PMetaData metaData, String name, TestTimeKeeper timeKeeper) throws TableNotFoundException { + PTable table = metaData.getTableRef(new PTableKey(null,name)).getTable(); + timeKeeper.incrementTime(); + return table; } private static void assertNames(PMetaData metaData, String... names) { @@ -56,50 +64,54 @@ private static class TestTimeKeeper implements TimeKeeper { @Override public long getCurrentTime() { - return time++; + return time; } + public void incrementTime() { + time++; + } } @Test public void testEviction() throws Exception { long maxSize = 10; - PMetaData metaData = new PMetaDataImpl(5, maxSize, new TestTimeKeeper()); - metaData = addToTable(metaData, "a", 5); + TestTimeKeeper timeKeeper = new TestTimeKeeper(); + PMetaData metaData = new PMetaDataImpl(5, maxSize, timeKeeper); + metaData = addToTable(metaData, "a", 5, timeKeeper); assertEquals(1, metaData.size()); - metaData = addToTable(metaData, "b", 4); + metaData = addToTable(metaData, "b", 4, timeKeeper); assertEquals(2, metaData.size()); - metaData = addToTable(metaData, "c", 3); + metaData = addToTable(metaData, "c", 3, timeKeeper); assertEquals(2, metaData.size()); assertNames(metaData, "b","c"); - metaData = addToTable(metaData, "b", 8); + metaData = addToTable(metaData, "b", 8, timeKeeper); assertEquals(1, metaData.size()); assertNames(metaData, "b"); - metaData = addToTable(metaData, "d", 11); + metaData = addToTable(metaData, "d", 11, timeKeeper); assertEquals(1, metaData.size()); assertNames(metaData, "d"); - metaData = removeFromTable(metaData, "d"); + metaData = removeFromTable(metaData, "d", timeKeeper); assertNames(metaData); - metaData = addToTable(metaData, "a", 4); + metaData = addToTable(metaData, "a", 4, timeKeeper); assertEquals(1, metaData.size()); - metaData = addToTable(metaData, "b", 3); + metaData = addToTable(metaData, "b", 3, timeKeeper); assertEquals(2, metaData.size()); - metaData = addToTable(metaData, "c", 2); + metaData = addToTable(metaData, "c", 2, timeKeeper); assertEquals(3, metaData.size()); assertNames(metaData, "a", "b","c"); - getFromTable(metaData, "a"); - metaData = addToTable(metaData, "d", 3); + getFromTable(metaData, "a", timeKeeper); + metaData = addToTable(metaData, "d", 3, timeKeeper); assertEquals(3, metaData.size()); assertNames(metaData, "c", "a","d"); // Clone maintains insert order metaData = metaData.clone(); - metaData = addToTable(metaData, "e", 6); + metaData = addToTable(metaData, "e", 6, timeKeeper); assertEquals(2, metaData.size()); assertNames(metaData, "d","e"); } @@ -107,18 +119,19 @@ public void testEviction() throws Exception { @Test public void shouldNotEvictMoreEntriesThanNecessary() throws Exception { long maxSize = 5; - PMetaData metaData = new PMetaDataImpl(5, maxSize, new TestTimeKeeper()); - metaData = addToTable(metaData, "a", 1); + TestTimeKeeper timeKeeper = new TestTimeKeeper(); + PMetaData metaData = new PMetaDataImpl(5, maxSize, timeKeeper); + metaData = addToTable(metaData, "a", 1, timeKeeper); assertEquals(1, metaData.size()); - metaData = addToTable(metaData, "b", 1); + metaData = addToTable(metaData, "b", 1, timeKeeper); assertEquals(2, metaData.size()); assertNames(metaData, "a", "b"); - metaData = addToTable(metaData, "c", 3); + metaData = addToTable(metaData, "c", 3, timeKeeper); assertEquals(3, metaData.size()); assertNames(metaData, "a", "b", "c"); - getFromTable(metaData, "a"); - getFromTable(metaData, "b"); - metaData = addToTable(metaData, "d", 3); + getFromTable(metaData, "a", timeKeeper); + getFromTable(metaData, "b", timeKeeper); + metaData = addToTable(metaData, "d", 3, timeKeeper); assertEquals(3, metaData.size()); assertNames(metaData, "a", "b", "d"); } @@ -126,19 +139,20 @@ public void shouldNotEvictMoreEntriesThanNecessary() throws Exception { @Test public void shouldAlwaysKeepAtLeastOneEntryEvenIfTooLarge() throws Exception { long maxSize = 5; - PMetaData metaData = new PMetaDataImpl(5, maxSize, new TestTimeKeeper()); - metaData = addToTable(metaData, "a", 1); + TestTimeKeeper timeKeeper = new TestTimeKeeper(); + PMetaData metaData = new PMetaDataImpl(5, maxSize, timeKeeper); + metaData = addToTable(metaData, "a", 1, timeKeeper); assertEquals(1, metaData.size()); - metaData = addToTable(metaData, "b", 1); + metaData = addToTable(metaData, "b", 1, timeKeeper); assertEquals(2, metaData.size()); - metaData = addToTable(metaData, "c", 5); + metaData = addToTable(metaData, "c", 5, timeKeeper); assertEquals(1, metaData.size()); - metaData = addToTable(metaData, "d", 20); + metaData = addToTable(metaData, "d", 20, timeKeeper); assertEquals(1, metaData.size()); assertNames(metaData, "d"); - metaData = addToTable(metaData, "e", 1); + metaData = addToTable(metaData, "e", 1, timeKeeper); assertEquals(1, metaData.size()); - metaData = addToTable(metaData, "f", 2); + metaData = addToTable(metaData, "f", 2, timeKeeper); assertEquals(2, metaData.size()); assertNames(metaData, "e", "f"); } @@ -146,23 +160,42 @@ public void shouldAlwaysKeepAtLeastOneEntryEvenIfTooLarge() throws Exception { @Test public void shouldAlwaysKeepOneEntryIfMaxSizeIsZero() throws Exception { long maxSize = 0; - PMetaData metaData = new PMetaDataImpl(0, maxSize, new TestTimeKeeper()); - metaData = addToTable(metaData, "a", 1); + TestTimeKeeper timeKeeper = new TestTimeKeeper(); + PMetaData metaData = new PMetaDataImpl(0, maxSize, timeKeeper); + metaData = addToTable(metaData, "a", 1, timeKeeper); assertEquals(1, metaData.size()); - metaData = addToTable(metaData, "b", 1); + metaData = addToTable(metaData, "b", 1, timeKeeper); assertEquals(1, metaData.size()); - metaData = addToTable(metaData, "c", 5); + metaData = addToTable(metaData, "c", 5, timeKeeper); assertEquals(1, metaData.size()); - metaData = addToTable(metaData, "d", 20); + metaData = addToTable(metaData, "d", 20, timeKeeper); assertEquals(1, metaData.size()); assertNames(metaData, "d"); - metaData = addToTable(metaData, "e", 1); + metaData = addToTable(metaData, "e", 1, timeKeeper); assertEquals(1, metaData.size()); - metaData = addToTable(metaData, "f", 2); + metaData = addToTable(metaData, "f", 2, timeKeeper); assertEquals(1, metaData.size()); assertNames(metaData, "f"); } + @Test + public void testAge() throws Exception { + long maxSize = 10; + TestTimeKeeper timeKeeper = new TestTimeKeeper(); + PMetaData metaData = new PMetaDataImpl(5, maxSize, timeKeeper); + String tableName = "a"; + metaData = addToTable(metaData, tableName, 1, timeKeeper); + PTableRef aTableRef = metaData.getTableRef(new PTableKey(null,tableName)); + assertNotNull(aTableRef); + assertEquals(1, metaData.getAge(aTableRef)); + tableName = "b"; + metaData = addToTable(metaData, tableName, 1, timeKeeper); + PTableRef bTableRef = metaData.getTableRef(new PTableKey(null,tableName)); + assertNotNull(bTableRef); + assertEquals(1, metaData.getAge(bTableRef)); + assertEquals(2, metaData.getAge(aTableRef)); + } + private static class PSizedTable extends PTableImpl { private final int size; private final PTableKey key; diff --git a/phoenix-protocol/src/main/PTable.proto b/phoenix-protocol/src/main/PTable.proto index 3048a40cd99..485dd11cdf9 100644 --- a/phoenix-protocol/src/main/PTable.proto +++ b/phoenix-protocol/src/main/PTable.proto @@ -87,4 +87,5 @@ message PTable { optional int32 baseColumnCount = 25; optional bool rowKeyOrderOptimizable = 26; optional bool transactional = 27; + optional int64 updateCacheFrequency = 28; } From 9a44b492673497e65e41ff45f3bcf7fb23982e35 Mon Sep 17 00:00:00 2001 From: James Taylor Date: Sun, 17 Jan 2016 12:11:04 -0800 Subject: [PATCH 19/47] PHOENIX-2593 Ensure all built-ins have required constructors (Dumindu Buddhika) --- .../expression/function/CeilFunction.java | 2 + .../expression/function/FloorFunction.java | 4 +- .../expression/function/NowFunction.java | 2 + .../expression/function/RoundFunction.java | 4 +- .../expression/function/ToCharFunction.java | 4 ++ .../expression/function/ToDateFunction.java | 4 ++ .../expression/function/ToNumberFunction.java | 4 ++ .../expression/function/ToTimeFunction.java | 4 ++ .../function/ToTimestampFunction.java | 4 ++ .../expression/function/TruncFunction.java | 2 + .../BuiltinFunctionConstructorTest.java | 65 +++++++++++++++++++ 11 files changed, 95 insertions(+), 4 deletions(-) create mode 100644 phoenix-core/src/test/java/org/apache/phoenix/expression/function/BuiltinFunctionConstructorTest.java diff --git a/phoenix-core/src/main/java/org/apache/phoenix/expression/function/CeilFunction.java b/phoenix-core/src/main/java/org/apache/phoenix/expression/function/CeilFunction.java index 706aadf8a2f..b65010a7536 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/expression/function/CeilFunction.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/expression/function/CeilFunction.java @@ -46,6 +46,8 @@ public abstract class CeilFunction extends ScalarFunction { public static final String NAME = "CEIL"; + + public CeilFunction() {} public CeilFunction(List children) { super(children); diff --git a/phoenix-core/src/main/java/org/apache/phoenix/expression/function/FloorFunction.java b/phoenix-core/src/main/java/org/apache/phoenix/expression/function/FloorFunction.java index 79383240aee..f6baa091489 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/expression/function/FloorFunction.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/expression/function/FloorFunction.java @@ -32,8 +32,6 @@ * * Base class for built-in FLOOR function. * - * - * @since 3.0.0 */ @BuiltInFunction(name = FloorFunction.NAME, nodeClass = FloorParseNode.class, @@ -47,6 +45,8 @@ public abstract class FloorFunction extends ScalarFunction { public static final String NAME = "FLOOR"; + public FloorFunction() {} + public FloorFunction(List children) { super(children); } diff --git a/phoenix-core/src/main/java/org/apache/phoenix/expression/function/NowFunction.java b/phoenix-core/src/main/java/org/apache/phoenix/expression/function/NowFunction.java index dc90249f636..83a5e5374f2 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/expression/function/NowFunction.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/expression/function/NowFunction.java @@ -35,6 +35,8 @@ public abstract class NowFunction extends ScalarFunction { public static final String NAME = "NOW"; + + public NowFunction() {} public NowFunction(List children) throws SQLException { super(children); diff --git a/phoenix-core/src/main/java/org/apache/phoenix/expression/function/RoundFunction.java b/phoenix-core/src/main/java/org/apache/phoenix/expression/function/RoundFunction.java index 9277716f3ea..7d0306c6926 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/expression/function/RoundFunction.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/expression/function/RoundFunction.java @@ -31,8 +31,6 @@ /** * Base class for RoundFunction. * - * - * @since 0.1 */ @BuiltInFunction(name = RoundFunction.NAME, nodeClass = RoundParseNode.class, @@ -46,6 +44,8 @@ public abstract class RoundFunction extends ScalarFunction { public static final String NAME = "ROUND"; + public RoundFunction() {} + public RoundFunction(List children) { super(children); } diff --git a/phoenix-core/src/main/java/org/apache/phoenix/expression/function/ToCharFunction.java b/phoenix-core/src/main/java/org/apache/phoenix/expression/function/ToCharFunction.java index e0bc68bff25..a14a0cbea98 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/expression/function/ToCharFunction.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/expression/function/ToCharFunction.java @@ -57,6 +57,10 @@ public class ToCharFunction extends ScalarFunction { public ToCharFunction() { } + public ToCharFunction(List children) throws SQLException { + this(children, null, null, null); + } + public ToCharFunction(List children, FunctionArgumentType type, String formatString, Format formatter) throws SQLException { super(children.subList(0, 1)); Preconditions.checkNotNull(formatString); diff --git a/phoenix-core/src/main/java/org/apache/phoenix/expression/function/ToDateFunction.java b/phoenix-core/src/main/java/org/apache/phoenix/expression/function/ToDateFunction.java index 01b0dfd5cf4..f41d9593b90 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/expression/function/ToDateFunction.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/expression/function/ToDateFunction.java @@ -59,6 +59,10 @@ public class ToDateFunction extends ScalarFunction { public ToDateFunction() { } + public ToDateFunction(List children) throws SQLException { + this(children, null, null); + } + public ToDateFunction(List children, String dateFormat, String timeZoneId) throws SQLException { super(children); init(dateFormat, timeZoneId); diff --git a/phoenix-core/src/main/java/org/apache/phoenix/expression/function/ToNumberFunction.java b/phoenix-core/src/main/java/org/apache/phoenix/expression/function/ToNumberFunction.java index ed70f2da041..e7659a1b2de 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/expression/function/ToNumberFunction.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/expression/function/ToNumberFunction.java @@ -60,6 +60,10 @@ public class ToNumberFunction extends ScalarFunction { public ToNumberFunction() {} + public ToNumberFunction(List children) throws SQLException { + this(children, null, null, null); + } + public ToNumberFunction(List children, FunctionArgumentType type, String formatString, Format formatter) throws SQLException { super(children.subList(0, 1)); Preconditions.checkNotNull(type); diff --git a/phoenix-core/src/main/java/org/apache/phoenix/expression/function/ToTimeFunction.java b/phoenix-core/src/main/java/org/apache/phoenix/expression/function/ToTimeFunction.java index 3a26584e278..2fa05533d4b 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/expression/function/ToTimeFunction.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/expression/function/ToTimeFunction.java @@ -47,6 +47,10 @@ public class ToTimeFunction extends ToDateFunction { public ToTimeFunction() { } + public ToTimeFunction(List children) throws SQLException { + this(children, null, null); + } + public ToTimeFunction(List children, String dateFormat, String timeZoneId) throws SQLException { super(children, dateFormat, timeZoneId); } diff --git a/phoenix-core/src/main/java/org/apache/phoenix/expression/function/ToTimestampFunction.java b/phoenix-core/src/main/java/org/apache/phoenix/expression/function/ToTimestampFunction.java index 17643a292fc..5a4828f50a1 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/expression/function/ToTimestampFunction.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/expression/function/ToTimestampFunction.java @@ -47,6 +47,10 @@ public class ToTimestampFunction extends ToDateFunction { public ToTimestampFunction() { } + public ToTimestampFunction(List children) throws SQLException { + this(children, null, null); + } + public ToTimestampFunction(List children, String dateFormat, String timeZoneId) throws SQLException { super(children, dateFormat, timeZoneId); } diff --git a/phoenix-core/src/main/java/org/apache/phoenix/expression/function/TruncFunction.java b/phoenix-core/src/main/java/org/apache/phoenix/expression/function/TruncFunction.java index a4e79c24473..3b9616cff4e 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/expression/function/TruncFunction.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/expression/function/TruncFunction.java @@ -51,6 +51,8 @@ public abstract class TruncFunction extends ScalarFunction { public static final String NAME = "TRUNC"; + + public TruncFunction() {} public TruncFunction(List children) throws SQLException { super(children); diff --git a/phoenix-core/src/test/java/org/apache/phoenix/expression/function/BuiltinFunctionConstructorTest.java b/phoenix-core/src/test/java/org/apache/phoenix/expression/function/BuiltinFunctionConstructorTest.java new file mode 100644 index 00000000000..5258283f72d --- /dev/null +++ b/phoenix-core/src/test/java/org/apache/phoenix/expression/function/BuiltinFunctionConstructorTest.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.expression.function; + +import java.lang.reflect.Constructor; +import java.util.ArrayList; +import java.util.List; + +import org.apache.phoenix.expression.function.ScalarFunction; +import org.apache.phoenix.expression.Expression; +import org.apache.phoenix.expression.ExpressionType; +import org.junit.Test; + +public class BuiltinFunctionConstructorTest { + + @Test + public void testChildrenListConstructors() { + ExpressionType[] types = ExpressionType.values(); + List children = new ArrayList<>(); + for(int i = 0; i < types.length; i++) { + try { + if((ScalarFunction.class.isAssignableFrom(types[i].getExpressionClass())) && (types[i].getExpressionClass() != UDFExpression.class)) { + Constructor cons = types[i].getExpressionClass().getDeclaredConstructor(List.class); + cons.setAccessible(true); + cons.newInstance(children); + } + } catch (NoSuchMethodException e) { + throw new RuntimeException(e); + } catch (Exception e) { + } + } + } + + @Test + public void testNoArgumentConstructors() { + ExpressionType[] types = ExpressionType.values(); + for(int i = 0; i < types.length; i++) { + try { + if(!AggregateFunction.class.isAssignableFrom(types[i].getExpressionClass())) { + Constructor cons = types[i].getExpressionClass().getDeclaredConstructor(); + cons.setAccessible(true); + cons.newInstance(); + } + } catch (NoSuchMethodException e) { + throw new RuntimeException(e); + } catch (Exception e) { + } + } + } +} From f591da44c9ee85ee7ab0fa910e3b18e649d86cdf Mon Sep 17 00:00:00 2001 From: James Taylor Date: Mon, 18 Jan 2016 21:09:44 -0800 Subject: [PATCH 20/47] PHOENIX-2591 Minimize transaction commit/rollback for DDL --- .../apache/phoenix/execute/MutationState.java | 30 +++++++++---------- .../apache/phoenix/util/TransactionUtil.java | 23 +++++--------- .../org/apache/phoenix/util/TestUtil.java | 2 +- 3 files changed, 24 insertions(+), 31 deletions(-) diff --git a/phoenix-core/src/main/java/org/apache/phoenix/execute/MutationState.java b/phoenix-core/src/main/java/org/apache/phoenix/execute/MutationState.java index 35a36e60141..ee694e7e393 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/execute/MutationState.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/execute/MutationState.java @@ -37,18 +37,6 @@ import javax.annotation.Nonnull; import javax.annotation.concurrent.Immutable; -import co.cask.tephra.Transaction; -import co.cask.tephra.Transaction.VisibilityLevel; -import co.cask.tephra.TransactionAware; -import co.cask.tephra.TransactionCodec; -import co.cask.tephra.TransactionConflictException; -import co.cask.tephra.TransactionContext; -import co.cask.tephra.TransactionFailureException; -import co.cask.tephra.TransactionSystemClient; -import co.cask.tephra.hbase11.TransactionAwareHTable; -import co.cask.tephra.visibility.FenceWait; -import co.cask.tephra.visibility.VisibilityFence; - import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.client.Delete; import org.apache.hadoop.hbase.client.HTableInterface; @@ -110,6 +98,18 @@ import com.google.common.collect.Maps; import com.google.common.collect.Sets; +import co.cask.tephra.Transaction; +import co.cask.tephra.Transaction.VisibilityLevel; +import co.cask.tephra.TransactionAware; +import co.cask.tephra.TransactionCodec; +import co.cask.tephra.TransactionConflictException; +import co.cask.tephra.TransactionContext; +import co.cask.tephra.TransactionFailureException; +import co.cask.tephra.TransactionSystemClient; +import co.cask.tephra.hbase11.TransactionAwareHTable; +import co.cask.tephra.visibility.FenceWait; +import co.cask.tephra.visibility.VisibilityFence; + /** * * Tracks the uncommitted state @@ -369,9 +369,9 @@ public boolean isTransactionStarted() { return getTransaction() != null; } - public long getReadPointer() { - Transaction tx = getTransaction(); - return tx == null ? HConstants.LATEST_TIMESTAMP : tx.getReadPointer(); + public long getInitialWritePointer() { + Transaction tx = getTransaction(); + return tx == null ? HConstants.LATEST_TIMESTAMP : tx.getTransactionId(); // First write pointer - won't change with checkpointing } // For testing diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/TransactionUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/TransactionUtil.java index 9f16e52297a..041c12eaa9d 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/util/TransactionUtil.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/util/TransactionUtil.java @@ -19,6 +19,11 @@ import java.sql.SQLException; +import co.cask.tephra.TransactionConflictException; +import co.cask.tephra.TransactionFailureException; +import co.cask.tephra.TxConstants; +import co.cask.tephra.hbase11.TransactionAwareHTable; + import org.apache.hadoop.hbase.client.HTableInterface; import org.apache.phoenix.coprocessor.MetaDataProtocol.MetaDataMutationResult; import org.apache.phoenix.exception.SQLExceptionCode; @@ -27,11 +32,6 @@ import org.apache.phoenix.jdbc.PhoenixConnection; import org.apache.phoenix.schema.PTable; -import co.cask.tephra.TransactionConflictException; -import co.cask.tephra.TransactionFailureException; -import co.cask.tephra.TxConstants; -import co.cask.tephra.hbase11.TransactionAwareHTable; - public class TransactionUtil { private TransactionUtil() { } @@ -67,7 +67,7 @@ public static TransactionAwareHTable getTransactionAwareHTable(HTableInterface h public static long getResolvedTimestamp(PhoenixConnection connection, boolean isTransactional, long defaultResolvedTimestamp) { MutationState mutationState = connection.getMutationState(); Long scn = connection.getSCN(); - return scn != null ? scn : (isTransactional && mutationState.isTransactionStarted()) ? convertToMilliseconds(mutationState.getReadPointer()) : defaultResolvedTimestamp; + return scn != null ? scn : (isTransactional && mutationState.isTransactionStarted()) ? convertToMilliseconds(mutationState.getInitialWritePointer()) : defaultResolvedTimestamp; } public static long getResolvedTime(PhoenixConnection connection, MetaDataMutationResult result) { @@ -80,7 +80,7 @@ public static long getResolvedTimestamp(PhoenixConnection connection, MetaDataMu PTable table = result.getTable(); MutationState mutationState = connection.getMutationState(); boolean txInProgress = table != null && table.isTransactional() && mutationState.isTransactionStarted(); - return txInProgress ? convertToMilliseconds(mutationState.getReadPointer()) : result.getMutationTime(); + return txInProgress ? convertToMilliseconds(mutationState.getInitialWritePointer()) : result.getMutationTime(); } public static Long getTableTimestamp(PhoenixConnection connection, boolean transactional) throws SQLException { @@ -89,17 +89,10 @@ public static Long getTableTimestamp(PhoenixConnection connection, boolean trans return timestamp; } MutationState mutationState = connection.getMutationState(); - // we need to burn a txn so that we are sure the txn read pointer is close to wall clock time if (!mutationState.isTransactionStarted()) { mutationState.startTransaction(); - connection.commit(); - } - else { - connection.commit(); } - mutationState.startTransaction(); - timestamp = convertToMilliseconds(mutationState.getReadPointer()); - connection.commit(); + timestamp = convertToMilliseconds(mutationState.getInitialWritePointer()); return timestamp; } } diff --git a/phoenix-core/src/test/java/org/apache/phoenix/util/TestUtil.java b/phoenix-core/src/test/java/org/apache/phoenix/util/TestUtil.java index e1258e41a50..66a3c65e46d 100644 --- a/phoenix-core/src/test/java/org/apache/phoenix/util/TestUtil.java +++ b/phoenix-core/src/test/java/org/apache/phoenix/util/TestUtil.java @@ -563,7 +563,7 @@ public static void analyzeTable(Connection conn, String tableName, boolean trans String query = "UPDATE STATISTICS " + tableName; conn.createStatement().execute(query); // if the table is transactional burn a txn in order to make sure the next txn read pointer is close to wall clock time - TransactionUtil.getTableTimestamp(conn.unwrap(PhoenixConnection.class), transactional); + conn.commit(); } public static void analyzeTableIndex(Connection conn, String tableName) throws IOException, SQLException { From 13699371820928cf14e0e2c5bbffe338c7aa2e93 Mon Sep 17 00:00:00 2001 From: James Taylor Date: Mon, 18 Jan 2016 21:14:34 -0800 Subject: [PATCH 21/47] PHOENIX-2478 Rows committed in transaction overlapping index creation are not populated --- .../apache/phoenix/execute/MutationState.java | 478 +++++++++--------- .../apache/phoenix/schema/MetaDataClient.java | 31 +- 2 files changed, 265 insertions(+), 244 deletions(-) diff --git a/phoenix-core/src/main/java/org/apache/phoenix/execute/MutationState.java b/phoenix-core/src/main/java/org/apache/phoenix/execute/MutationState.java index ee694e7e393..a6fe98d1ea6 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/execute/MutationState.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/execute/MutationState.java @@ -121,6 +121,7 @@ public class MutationState implements SQLCloseable { private static final Logger logger = LoggerFactory.getLogger(MutationState.class); private static final TransactionCodec CODEC = new TransactionCodec(); private static final int[] EMPTY_STATEMENT_INDEX_ARRAY = new int[0]; + private static final int MAX_COMMIT_RETRIES = 3; private final PhoenixConnection connection; private final long maxSize; @@ -160,37 +161,37 @@ private MutationState(long maxSize, PhoenixConnection connection, Transaction tx } private MutationState(long maxSize, PhoenixConnection connection, Transaction tx, TransactionContext txContext, long sizeOffset) { - this(maxSize, connection, Maps.>newHashMapWithExpectedSize(5), tx, txContext); + this(maxSize, connection, Maps.>newHashMapWithExpectedSize(5), tx, txContext); this.sizeOffset = sizeOffset; } - MutationState(long maxSize, PhoenixConnection connection, - Map> mutations, - Transaction tx, TransactionContext txContext) { - this.maxSize = maxSize; - this.connection = connection; - this.mutations = mutations; - boolean isMetricsEnabled = connection.isRequestLevelMetricsEnabled(); - this.mutationMetricQueue = isMetricsEnabled ? new MutationMetricQueue() - : NoOpMutationMetricsQueue.NO_OP_MUTATION_METRICS_QUEUE; - this.tx = tx; - if (tx == null) { + MutationState(long maxSize, PhoenixConnection connection, + Map> mutations, + Transaction tx, TransactionContext txContext) { + this.maxSize = maxSize; + this.connection = connection; + this.mutations = mutations; + boolean isMetricsEnabled = connection.isRequestLevelMetricsEnabled(); + this.mutationMetricQueue = isMetricsEnabled ? new MutationMetricQueue() + : NoOpMutationMetricsQueue.NO_OP_MUTATION_METRICS_QUEUE; + this.tx = tx; + if (tx == null) { this.txAwares = Collections.emptyList(); - if (txContext == null) { - TransactionSystemClient txServiceClient = this.connection - .getQueryServices().getTransactionSystemClient(); - this.txContext = new TransactionContext(txServiceClient); - } else { - isExternalTxContext = true; - this.txContext = txContext; - } - } else { - // this code path is only used while running child scans, we can't pass the txContext to child scans - // as it is not thread safe, so we use the tx member variable - this.txAwares = Lists.newArrayList(); - this.txContext = null; - } - } + if (txContext == null) { + TransactionSystemClient txServiceClient = this.connection + .getQueryServices().getTransactionSystemClient(); + this.txContext = new TransactionContext(txServiceClient); + } else { + isExternalTxContext = true; + this.txContext = txContext; + } + } else { + // this code path is only used while running child scans, we can't pass the txContext to child scans + // as it is not thread safe, so we use the tx member variable + this.txAwares = Lists.newArrayList(); + this.txContext = null; + } + } public MutationState(TableRef table, Map mutations, long sizeOffset, long maxSize, PhoenixConnection connection) { this(maxSize, connection, null, null, sizeOffset); @@ -217,9 +218,11 @@ public long getMaxSize() { public void commitWriteFence(PTable dataTable) throws SQLException { if (dataTable.isTransactional()) { byte[] key = SchemaUtil.getTableKey(dataTable); + boolean success = false; try { FenceWait fenceWait = VisibilityFence.prepareWait(key, connection.getQueryServices().getTransactionSystemClient()); fenceWait.await(10000, TimeUnit.MILLISECONDS); + success = true; } catch (InterruptedException e) { Thread.currentThread().interrupt(); throw new SQLExceptionInfo.Builder(SQLExceptionCode.INTERRUPTED_EXCEPTION).setRootCause(e).build().buildException(); @@ -235,6 +238,7 @@ public void commitWriteFence(PTable dataTable) throws SQLException { // TODO: seems like an autonomous tx capability in Tephra would be useful here. try { txContext.start(); + if (logger.isInfoEnabled() && success) logger.info("Added write fence at ~" + getTransaction().getReadPointer()); } catch (TransactionFailureException e) { throw TransactionUtil.getTransactionFailureException(e); } @@ -306,18 +310,18 @@ public boolean checkpointIfNeccessary(MutationPlan plan) throws SQLException { } if (hasUncommittedData) { try { - if (txContext == null) { - currentTx = tx = connection.getQueryServices().getTransactionSystemClient().checkpoint(currentTx); - } else { - txContext.checkpoint(); - currentTx = tx = txContext.getCurrentTransaction(); - } + if (txContext == null) { + currentTx = tx = connection.getQueryServices().getTransactionSystemClient().checkpoint(currentTx); + } else { + txContext.checkpoint(); + currentTx = tx = txContext.getCurrentTransaction(); + } // Since we've checkpointed, we can clear out uncommitted set, since a statement run afterwards // should see all this data. uncommittedPhysicalNames.clear(); } catch (TransactionFailureException e) { throw new SQLException(e); - } + } } // Since we're querying our own table while mutating it, we must exclude // see our current mutations, otherwise we can get erroneous results (for DELETE) @@ -356,7 +360,7 @@ public HTableInterface getHTable(PTable table) throws SQLException { } public PhoenixConnection getConnection() { - return connection; + return connection; } // Kept private as the Transaction may change when check pointed. Keeping it private ensures @@ -366,7 +370,7 @@ private Transaction getTransaction() { } public boolean isTransactionStarted() { - return getTransaction() != null; + return getTransaction() != null; } public long getInitialWritePointer() { @@ -391,11 +395,11 @@ public boolean startTransaction() throws SQLException { throw new SQLExceptionInfo.Builder(SQLExceptionCode.NULL_TRANSACTION_CONTEXT).build().buildException(); } - if (connection.getSCN() != null) { - throw new SQLExceptionInfo.Builder( - SQLExceptionCode.CANNOT_START_TRANSACTION_WITH_SCN_SET) - .build().buildException(); - } + if (connection.getSCN() != null) { + throw new SQLExceptionInfo.Builder( + SQLExceptionCode.CANNOT_START_TRANSACTION_WITH_SCN_SET) + .build().buildException(); + } try { if (!isTransactionStarted()) { @@ -460,9 +464,9 @@ public void join(MutationState newMutationState) { // Loop through new rows and replace existing with new for (Map.Entry rowEntry : entry.getValue().entrySet()) { // Replace existing row with new row - RowMutationState existingRowMutationState = existingRows.put(rowEntry.getKey(), rowEntry.getValue()); + RowMutationState existingRowMutationState = existingRows.put(rowEntry.getKey(), rowEntry.getValue()); if (existingRowMutationState != null) { - Map existingValues = existingRowMutationState.getColumnValues(); + Map existingValues = existingRowMutationState.getColumnValues(); if (existingValues != PRow.DELETE_MARKER) { Map newRow = rowEntry.getValue().getColumnValues(); // if new row is PRow.DELETE_MARKER, it means delete, and we don't need to merge it with existing row. @@ -502,7 +506,7 @@ public void join(MutationState newMutationState) { } - private static ImmutableBytesPtr getNewRowKeyWithRowTimestamp(ImmutableBytesPtr ptr, long rowTimestamp, PTable table) { + private static ImmutableBytesPtr getNewRowKeyWithRowTimestamp(ImmutableBytesPtr ptr, long rowTimestamp, PTable table) { RowKeySchema schema = table.getRowKeySchema(); int rowTimestampColPos = table.getRowTimestampColPos(); Field rowTimestampField = schema.getField(rowTimestampColPos); @@ -523,7 +527,7 @@ private static ImmutableBytesPtr getNewRowKeyWithRowTimestamp(ImmutableBytesPtr return ptr; } - private Iterator>> addRowMutations(final TableRef tableRef, final Map values, final long timestamp, boolean includeMutableIndexes, final boolean sendAll) { + private Iterator>> addRowMutations(final TableRef tableRef, final Map values, final long timestamp, boolean includeMutableIndexes, final boolean sendAll) { final PTable table = tableRef.getTable(); final Iterator indexes = // Only maintain tables with immutable rows through this client-side mechanism (table.isImmutableRows() || includeMutableIndexes) ? @@ -554,13 +558,13 @@ public Pair> next() { connection.getKeyValueBuilder(), connection); // we may also have to include delete mutations for immutable tables if we are not processing all the tables in the mutations map if (!sendAll) { - TableRef key = new TableRef(index); - Map rowToColumnMap = mutations.remove(key); - if (rowToColumnMap!=null) { - final List deleteMutations = Lists.newArrayList(); - generateMutations(tableRef, timestamp, rowToColumnMap, deleteMutations, null); - indexMutations.addAll(deleteMutations); - } + TableRef key = new TableRef(index); + Map rowToColumnMap = mutations.remove(key); + if (rowToColumnMap!=null) { + final List deleteMutations = Lists.newArrayList(); + generateMutations(tableRef, timestamp, rowToColumnMap, deleteMutations, null); + indexMutations.addAll(deleteMutations); + } } } catch (SQLException e) { throw new IllegalDataException(e); @@ -676,32 +680,32 @@ public void remove() { }; } - /** - * Validates that the meta data is valid against the server meta data if we haven't yet done so. - * Otherwise, for every UPSERT VALUES call, we'd need to hit the server to see if the meta data - * has changed. - * @param connection - * @return the server time to use for the upsert - * @throws SQLException if the table or any columns no longer exist - */ - private long[] validateAll() throws SQLException { - int i = 0; - long[] timeStamps = new long[this.mutations.size()]; - for (Map.Entry> entry : mutations.entrySet()) { - TableRef tableRef = entry.getKey(); - timeStamps[i++] = validate(tableRef, entry.getValue()); - } - return timeStamps; - } - - private long validate(TableRef tableRef, Map rowKeyToColumnMap) throws SQLException { - Long scn = connection.getSCN(); - MetaDataClient client = new MetaDataClient(connection); - long serverTimeStamp = tableRef.getTimeStamp(); - // If we're auto committing, we've already validated the schema when we got the ColumnResolver, - // so no need to do it again here. - if (!connection.getAutoCommit()) { - PTable table = tableRef.getTable(); + /** + * Validates that the meta data is valid against the server meta data if we haven't yet done so. + * Otherwise, for every UPSERT VALUES call, we'd need to hit the server to see if the meta data + * has changed. + * @param connection + * @return the server time to use for the upsert + * @throws SQLException if the table or any columns no longer exist + */ + private long[] validateAll() throws SQLException { + int i = 0; + long[] timeStamps = new long[this.mutations.size()]; + for (Map.Entry> entry : mutations.entrySet()) { + TableRef tableRef = entry.getKey(); + timeStamps[i++] = validate(tableRef, entry.getValue()); + } + return timeStamps; + } + + private long validate(TableRef tableRef, Map rowKeyToColumnMap) throws SQLException { + Long scn = connection.getSCN(); + MetaDataClient client = new MetaDataClient(connection); + long serverTimeStamp = tableRef.getTimeStamp(); + // If we're auto committing, we've already validated the schema when we got the ColumnResolver, + // so no need to do it again here. + if (!connection.getAutoCommit()) { + PTable table = tableRef.getTable(); MetaDataMutationResult result = client.updateCache(table.getSchemaName().getString(), table.getTableName().getString()); PTable resolvedTable = result.getTable(); if (resolvedTable == null) { @@ -717,14 +721,14 @@ private long validate(TableRef tableRef, Map rowEntry : rowKeyToColumnMap.entrySet()) { - RowMutationState valueEntry = rowEntry.getValue(); + RowMutationState valueEntry = rowEntry.getValue(); if (valueEntry != null) { - Map colValues = valueEntry.getColumnValues(); - if (colValues != PRow.DELETE_MARKER) { + Map colValues = valueEntry.getColumnValues(); + if (colValues != PRow.DELETE_MARKER) { for (PColumn column : colValues.keySet()) { columns[column.getPosition()] = column; } - } + } } } for (PColumn column : columns) { @@ -735,8 +739,8 @@ private long validate(TableRef tableRef, Map mutations) { long byteSize = 0; @@ -845,74 +849,74 @@ private void send(Iterator tableRefIterator) throws SQLException { // add tracing for this operation try (TraceScope trace = Tracing.startNewSpan(connection, "Committing mutations to tables")) { Span span = trace.getSpan(); - ImmutableBytesWritable indexMetaDataPtr = new ImmutableBytesWritable(); - boolean isTransactional; - while (tableRefIterator.hasNext()) { - // at this point we are going through mutations for each table - final TableRef tableRef = tableRefIterator.next(); - valuesMap = mutations.get(tableRef); - if (valuesMap == null || valuesMap.isEmpty()) { - continue; - } + ImmutableBytesWritable indexMetaDataPtr = new ImmutableBytesWritable(); + boolean isTransactional; + while (tableRefIterator.hasNext()) { + // at this point we are going through mutations for each table + final TableRef tableRef = tableRefIterator.next(); + valuesMap = mutations.get(tableRef); + if (valuesMap == null || valuesMap.isEmpty()) { + continue; + } // Validate as we go if transactional since we can undo if a problem occurs (which is unlikely) long serverTimestamp = serverTimeStamps == null ? validate(tableRef, valuesMap) : serverTimeStamps[i++]; - final PTable table = tableRef.getTable(); - // Track tables to which we've sent uncommitted data - if (isTransactional = table.isTransactional()) { - txTableRefs.add(tableRef); - uncommittedPhysicalNames.add(table.getPhysicalName().getString()); - } - boolean isDataTable = true; + final PTable table = tableRef.getTable(); + // Track tables to which we've sent uncommitted data + if (isTransactional = table.isTransactional()) { + txTableRefs.add(tableRef); + uncommittedPhysicalNames.add(table.getPhysicalName().getString()); + } + boolean isDataTable = true; table.getIndexMaintainers(indexMetaDataPtr, connection); - Iterator>> mutationsIterator = addRowMutations(tableRef, valuesMap, serverTimestamp, false, sendAll); - while (mutationsIterator.hasNext()) { - Pair> pair = mutationsIterator.next(); - byte[] htableName = pair.getFirst(); - List mutationList = pair.getSecond(); - - //create a span per target table - //TODO maybe we can be smarter about the table name to string here? - Span child = Tracing.child(span,"Writing mutation batch for table: "+Bytes.toString(htableName)); - - int retryCount = 0; - boolean shouldRetry = false; - do { - final ServerCache cache = isDataTable ? setMetaDataOnMutations(tableRef, mutationList, indexMetaDataPtr) : null; - - // If we haven't retried yet, retry for this case only, as it's possible that - // a split will occur after we send the index metadata cache to all known - // region servers. - shouldRetry = cache != null; - SQLException sqlE = null; - HTableInterface hTable = connection.getQueryServices().getTable(htableName); - try { - if (isTransactional) { - // If we have indexes, wrap the HTable in a delegate HTable that - // will attach the necessary index meta data in the event of a - // rollback - if (!table.getIndexes().isEmpty()) { - hTable = new MetaDataAwareHTable(hTable, tableRef); - } - TransactionAwareHTable txnAware = TransactionUtil.getTransactionAwareHTable(hTable, table); - // Don't add immutable indexes (those are the only ones that would participate - // during a commit), as we don't need conflict detection for these. - if (isDataTable) { - // Even for immutable, we need to do this so that an abort has the state - // necessary to generate the rows to delete. - addTransactionParticipant(txnAware); - } else { - txnAware.startTx(getTransaction()); - } - hTable = txnAware; - } - long numMutations = mutationList.size(); + Iterator>> mutationsIterator = addRowMutations(tableRef, valuesMap, serverTimestamp, false, sendAll); + while (mutationsIterator.hasNext()) { + Pair> pair = mutationsIterator.next(); + byte[] htableName = pair.getFirst(); + List mutationList = pair.getSecond(); + + //create a span per target table + //TODO maybe we can be smarter about the table name to string here? + Span child = Tracing.child(span,"Writing mutation batch for table: "+Bytes.toString(htableName)); + + int retryCount = 0; + boolean shouldRetry = false; + do { + final ServerCache cache = isDataTable ? setMetaDataOnMutations(tableRef, mutationList, indexMetaDataPtr) : null; + + // If we haven't retried yet, retry for this case only, as it's possible that + // a split will occur after we send the index metadata cache to all known + // region servers. + shouldRetry = cache != null; + SQLException sqlE = null; + HTableInterface hTable = connection.getQueryServices().getTable(htableName); + try { + if (isTransactional) { + // If we have indexes, wrap the HTable in a delegate HTable that + // will attach the necessary index meta data in the event of a + // rollback + if (!table.getIndexes().isEmpty()) { + hTable = new MetaDataAwareHTable(hTable, tableRef); + } + TransactionAwareHTable txnAware = TransactionUtil.getTransactionAwareHTable(hTable, table); + // Don't add immutable indexes (those are the only ones that would participate + // during a commit), as we don't need conflict detection for these. + if (isDataTable) { + // Even for immutable, we need to do this so that an abort has the state + // necessary to generate the rows to delete. + addTransactionParticipant(txnAware); + } else { + txnAware.startTx(getTransaction()); + } + hTable = txnAware; + } + long numMutations = mutationList.size(); GLOBAL_MUTATION_BATCH_SIZE.update(numMutations); long startTime = System.currentTimeMillis(); child.addTimelineAnnotation("Attempt " + retryCount); - hTable.batch(mutationList); - child.stop(); - child.stop(); + hTable.batch(mutationList); + child.stop(); + child.stop(); shouldRetry = false; long mutationCommitTime = System.currentTimeMillis() - startTime; GLOBAL_MUTATION_COMMIT_TIME.update(mutationCommitTime); @@ -920,80 +924,80 @@ private void send(Iterator tableRefIterator) throws SQLException { long mutationSizeBytes = calculateMutationSize(mutationList); MutationMetric mutationsMetric = new MutationMetric(numMutations, mutationSizeBytes, mutationCommitTime); mutationMetricQueue.addMetricsForTable(Bytes.toString(htableName), mutationsMetric); - } catch (Exception e) { - SQLException inferredE = ServerUtil.parseServerExceptionOrNull(e); - if (inferredE != null) { - if (shouldRetry && retryCount == 0 && inferredE.getErrorCode() == SQLExceptionCode.INDEX_METADATA_NOT_FOUND.getErrorCode()) { - // Swallow this exception once, as it's possible that we split after sending the index metadata - // and one of the region servers doesn't have it. This will cause it to have it the next go around. - // If it fails again, we don't retry. - String msg = "Swallowing exception and retrying after clearing meta cache on connection. " + inferredE; - logger.warn(LogUtil.addCustomAnnotations(msg, connection)); - connection.getQueryServices().clearTableRegionCache(htableName); - - // add a new child span as this one failed - child.addTimelineAnnotation(msg); - child.stop(); - child = Tracing.child(span,"Failed batch, attempting retry"); - - continue; - } - e = inferredE; - } - // Throw to client an exception that indicates the statements that - // were not committed successfully. - sqlE = new CommitException(e, getUncommittedStatementIndexes()); - } finally { - try { - if (cache != null) { - cache.close(); - } - } finally { - try { - hTable.close(); - } - catch (IOException e) { - if (sqlE != null) { - sqlE.setNextException(ServerUtil.parseServerException(e)); - } else { - sqlE = ServerUtil.parseServerException(e); - } - } - if (sqlE != null) { - throw sqlE; - } - } - } - } while (shouldRetry && retryCount++ < 1); - isDataTable = false; - } - if (tableRef.getTable().getType() != PTableType.INDEX) { - numRows -= valuesMap.size(); - } - // For transactions, track the statement indexes as we send data - // over because our CommitException should include all statements - // involved in the transaction since none of them would have been - // committed in the event of a failure. - if (isTransactional) { - addUncommittedStatementIndexes(valuesMap.values()); - if (txMutations == null) { - txMutations = Maps.newHashMapWithExpectedSize(mutations.size()); - } - // Keep all mutations we've encountered until a commit or rollback. - // This is not ideal, but there's not good way to get the values back - // in the event that we need to replay the commit. - txMutations.put(tableRef, valuesMap); - } + } catch (Exception e) { + SQLException inferredE = ServerUtil.parseServerExceptionOrNull(e); + if (inferredE != null) { + if (shouldRetry && retryCount == 0 && inferredE.getErrorCode() == SQLExceptionCode.INDEX_METADATA_NOT_FOUND.getErrorCode()) { + // Swallow this exception once, as it's possible that we split after sending the index metadata + // and one of the region servers doesn't have it. This will cause it to have it the next go around. + // If it fails again, we don't retry. + String msg = "Swallowing exception and retrying after clearing meta cache on connection. " + inferredE; + logger.warn(LogUtil.addCustomAnnotations(msg, connection)); + connection.getQueryServices().clearTableRegionCache(htableName); + + // add a new child span as this one failed + child.addTimelineAnnotation(msg); + child.stop(); + child = Tracing.child(span,"Failed batch, attempting retry"); + + continue; + } + e = inferredE; + } + // Throw to client an exception that indicates the statements that + // were not committed successfully. + sqlE = new CommitException(e, getUncommittedStatementIndexes()); + } finally { + try { + if (cache != null) { + cache.close(); + } + } finally { + try { + hTable.close(); + } + catch (IOException e) { + if (sqlE != null) { + sqlE.setNextException(ServerUtil.parseServerException(e)); + } else { + sqlE = ServerUtil.parseServerException(e); + } + } + if (sqlE != null) { + throw sqlE; + } + } + } + } while (shouldRetry && retryCount++ < 1); + isDataTable = false; + } + if (tableRef.getTable().getType() != PTableType.INDEX) { + numRows -= valuesMap.size(); + } + // For transactions, track the statement indexes as we send data + // over because our CommitException should include all statements + // involved in the transaction since none of them would have been + // committed in the event of a failure. + if (isTransactional) { + addUncommittedStatementIndexes(valuesMap.values()); + if (txMutations == null) { + txMutations = Maps.newHashMapWithExpectedSize(mutations.size()); + } + // Keep all mutations we've encountered until a commit or rollback. + // This is not ideal, but there's not good way to get the values back + // in the event that we need to replay the commit. + txMutations.put(tableRef, valuesMap); + } // Remove batches as we process them - if (sendAll) { - // Iterating through map key set in this case, so we cannot use - // the remove method without getting a concurrent modification - // exception. - tableRefIterator.remove(); - } else { - mutations.remove(tableRef); - } - } + if (sendAll) { + // Iterating through map key set in this case, so we cannot use + // the remove method without getting a concurrent modification + // exception. + tableRefIterator.remove(); + } else { + mutations.remove(tableRef); + } + } } } @@ -1006,7 +1010,7 @@ public byte[] encodeTransaction() throws SQLException { } public static Transaction decodeTransaction(byte[] txnBytes) throws IOException { - return (txnBytes == null || txnBytes.length==0) ? null : CODEC.decode(txnBytes); + return (txnBytes == null || txnBytes.length==0) ? null : CODEC.decode(txnBytes); } private ServerCache setMetaDataOnMutations(TableRef tableRef, List mutations, @@ -1059,10 +1063,10 @@ private void addUncommittedStatementIndexes(Collection rowMuta } private int[] getUncommittedStatementIndexes() { - for (Map rowMutationMap : mutations.values()) { - addUncommittedStatementIndexes(rowMutationMap.values()); - } - return uncommittedStatementIndexes; + for (Map rowMutationMap : mutations.values()) { + addUncommittedStatementIndexes(rowMutationMap.values()); + } + return uncommittedStatementIndexes; } @Override @@ -1101,9 +1105,9 @@ public void commit() throws SQLException { Map> txMutations = Collections.emptyMap(); int retryCount = 0; do { - boolean sendSuccessful=false; - boolean retryCommit = false; - SQLException sqlE = null; + boolean sendSuccessful=false; + boolean retryCommit = false; + SQLException sqlE = null; try { send(); txMutations = this.txMutations; @@ -1121,7 +1125,8 @@ public void commit() throws SQLException { finishSuccessful = true; } } catch (TransactionFailureException e) { - retryCommit = (e instanceof TransactionConflictException && retryCount == 0); + if (logger.isInfoEnabled()) logger.info(e.getClass().getName() + " at timestamp " + getInitialWritePointer() + " with retry count of " + retryCount); + retryCommit = (e instanceof TransactionConflictException && retryCount < MAX_COMMIT_RETRIES); txFailure = e; SQLException nextE = TransactionUtil.getTransactionFailureException(e); if (sqlE == null) { @@ -1134,7 +1139,9 @@ public void commit() throws SQLException { if (!finishSuccessful) { try { txContext.abort(txFailure); + if (logger.isInfoEnabled()) logger.info("Abort successful"); } catch (TransactionFailureException e) { + if (logger.isInfoEnabled()) logger.info("Abort failed with " + e); SQLException nextE = TransactionUtil.getTransactionFailureException(e); if (sqlE == null) { sqlE = nextE; @@ -1151,8 +1158,15 @@ public void commit() throws SQLException { } finally { if (retryCommit) { startTransaction(); + // Add back read fences + Set txTableRefs = txMutations.keySet(); + for (TableRef tableRef : txTableRefs) { + PTable dataTable = tableRef.getTable(); + addReadFence(dataTable); + } try { - retryCommit = wasIndexAdded(txMutations.keySet()); + // Only retry if an index was added + retryCommit = wasIndexAdded(txTableRefs); } catch (SQLException e) { retryCommit = false; if (sqlE == null) { @@ -1173,7 +1187,9 @@ public void commit() throws SQLException { break; } retryCount++; - mutations.putAll(txMutations); + if (txMutations != null) { + mutations.putAll(txMutations); + } } while (true); } @@ -1183,6 +1199,7 @@ public void commit() throws SQLException { * @throws SQLException */ private boolean wasIndexAdded(Set txTableRefs) throws SQLException { + if (logger.isInfoEnabled()) logger.info("Checking for index updates as of " + getInitialWritePointer()); MetaDataClient client = new MetaDataClient(connection); PMetaData cache = connection.getMetaDataCache(); boolean addedIndexes = false; @@ -1198,6 +1215,7 @@ private boolean wasIndexAdded(Set txTableRefs) throws SQLException { throw new TableNotFoundException(dataTable.getSchemaName().getString(), dataTable.getTableName().getString()); } if (!result.wasUpdated()) { + if (logger.isInfoEnabled()) logger.info("No updates to " + dataTable.getName().getString() + " as of " + timestamp); continue; } if (!addedIndexes) { @@ -1205,8 +1223,10 @@ private boolean wasIndexAdded(Set txTableRefs) throws SQLException { // that an index was dropped and recreated with the same name but different // indexed/covered columns. addedIndexes = (!oldIndexes.equals(result.getTable().getIndexes())); + if (logger.isInfoEnabled()) logger.info((addedIndexes ? "Updates " : "No updates ") + "as of " + timestamp + " to " + dataTable.getName().getString() + " with indexes " + dataTable.getIndexes()); } } + if (logger.isInfoEnabled()) logger.info((addedIndexes ? "Updates " : "No updates ") + "to indexes as of " + getInitialWritePointer()); return addedIndexes; } diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java index ee212ed4ac1..d134f0898a1 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java @@ -479,8 +479,8 @@ private MetaDataMutationResult updateCache(PName tenantId, String schemaName, St // Do not make rpc to getTable if // 1. table is a system table // 2. table was already resolved as of that timestamp - if (table != null && !alwaysHitServer - && (systemTable || resolvedTimestamp == tableResolvedTimestamp || connection.getMetaDataCache().getAge(tableRef) < table.getUpdateCacheFrequency())) { + if (table != null && !alwaysHitServer + && (systemTable || resolvedTimestamp == tableResolvedTimestamp || connection.getMetaDataCache().getAge(tableRef) < table.getUpdateCacheFrequency())) { return new MetaDataMutationResult(MutationCode.TABLE_ALREADY_EXISTS, QueryConstants.UNSET_TIMESTAMP, table); } @@ -1383,6 +1383,7 @@ public MutationState createIndex(CreateIndexStatement statement, byte[][] splits return new MutationState(0,connection); } + if (logger.isInfoEnabled()) logger.info("Created index " + table.getName().getString() + " at " + table.getTimeStamp()); // In async process, we return immediately as the MR job needs to be triggered . if(statement.isAsync()) { return new MutationState(0, connection); @@ -2887,19 +2888,19 @@ public MutationState addColumn(AddColumnStatement statement) throws SQLException String fullTableName = SchemaUtil.getTableName(schemaName, tableName); long resolvedTimeStamp = TransactionUtil.getResolvedTime(connection, result); if (table.getIndexes().isEmpty() || (numPkColumnsAdded==0 && !nonTxToTx)) { - connection.addColumn( - tenantId, - fullTableName, - columns, - result.getMutationTime(), - seqNum, - isImmutableRows == null ? table.isImmutableRows() : isImmutableRows, - disableWAL == null ? table.isWALDisabled() : disableWAL, - multiTenant == null ? table.isMultiTenant() : multiTenant, - storeNulls == null ? table.getStoreNulls() : storeNulls, - isTransactional == null ? table.isTransactional() : isTransactional, - updateCacheFrequency == null ? table.getUpdateCacheFrequency() : updateCacheFrequency, - resolvedTimeStamp); + connection.addColumn( + tenantId, + fullTableName, + columns, + result.getMutationTime(), + seqNum, + isImmutableRows == null ? table.isImmutableRows() : isImmutableRows, + disableWAL == null ? table.isWALDisabled() : disableWAL, + multiTenant == null ? table.isMultiTenant() : multiTenant, + storeNulls == null ? table.getStoreNulls() : storeNulls, + isTransactional == null ? table.isTransactional() : isTransactional, + updateCacheFrequency == null ? table.getUpdateCacheFrequency() : updateCacheFrequency, + resolvedTimeStamp); } else if (updateCacheFrequency != null) { // Force removal from cache as the update cache frequency has changed // Note that clients outside this JVM won't be affected. From 3520e12858223dfcad343e0a3a29c71fe4d074bc Mon Sep 17 00:00:00 2001 From: James Taylor Date: Tue, 19 Jan 2016 08:19:44 -0800 Subject: [PATCH 22/47] PHOENIX-2478 Rows committed in transaction overlapping index creation are not populated --- .../end2end/index/ImmutableIndexIT.java | 239 ++++++++++++++---- .../apache/phoenix/execute/MutationState.java | 115 +++++---- .../org/apache/phoenix/query/BaseTest.java | 103 +++----- 3 files changed, 285 insertions(+), 172 deletions(-) diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/ImmutableIndexIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/ImmutableIndexIT.java index 0d329fefdd8..c4ecfbb627f 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/ImmutableIndexIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/ImmutableIndexIT.java @@ -20,7 +20,6 @@ import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; import java.sql.Connection; import java.sql.DriverManager; @@ -29,12 +28,26 @@ import java.sql.Statement; import java.util.Arrays; import java.util.Collection; +import java.util.List; import java.util.Map; import java.util.Properties; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import org.apache.hadoop.hbase.DoNotRetryIOException; +import org.apache.hadoop.hbase.HBaseIOException; +import org.apache.hadoop.hbase.client.Durability; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.coprocessor.ObserverContext; +import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment; +import org.apache.hadoop.hbase.coprocessor.SimpleRegionObserver; +import org.apache.hadoop.hbase.regionserver.wal.WALEdit; +import org.apache.hadoop.hbase.util.Bytes; import org.apache.phoenix.end2end.BaseHBaseManagedTimeIT; import org.apache.phoenix.end2end.Shadower; -import org.apache.phoenix.exception.SQLExceptionCode; import org.apache.phoenix.query.BaseTest; import org.apache.phoenix.query.QueryServices; import org.apache.phoenix.util.PropertiesUtil; @@ -47,80 +60,194 @@ import org.junit.runners.Parameterized; import org.junit.runners.Parameterized.Parameters; +import com.google.common.collect.Lists; import com.google.common.collect.Maps; @RunWith(Parameterized.class) public class ImmutableIndexIT extends BaseHBaseManagedTimeIT { - - private final boolean localIndex; - private final String tableDDLOptions; - private final String tableName; + + private final boolean localIndex; + private final String tableDDLOptions; + private final String tableName; private final String indexName; private final String fullTableName; private final String fullIndexName; - - public ImmutableIndexIT(boolean localIndex, boolean transactional) { - this.localIndex = localIndex; - StringBuilder optionBuilder = new StringBuilder("IMMUTABLE_ROWS=true"); - if (transactional) { - optionBuilder.append(", TRANSACTIONAL=true"); - } - this.tableDDLOptions = optionBuilder.toString(); - this.tableName = TestUtil.DEFAULT_DATA_TABLE_NAME + ( transactional ? "_TXN" : ""); + + private static String TABLE_NAME; + private static String INDEX_DDL; + public static final AtomicInteger NUM_ROWS = new AtomicInteger(1); + + public ImmutableIndexIT(boolean localIndex, boolean transactional) { + this.localIndex = localIndex; + StringBuilder optionBuilder = new StringBuilder("IMMUTABLE_ROWS=true"); + if (transactional) { + optionBuilder.append(", TRANSACTIONAL=true"); + } + this.tableDDLOptions = optionBuilder.toString(); + this.tableName = TestUtil.DEFAULT_DATA_TABLE_NAME + ( transactional ? "_TXN" : ""); this.indexName = "IDX" + ( transactional ? "_TXN" : ""); this.fullTableName = SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, tableName); this.fullIndexName = SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, indexName); - } - - @BeforeClass + } + + @BeforeClass @Shadower(classBeingShadowed = BaseHBaseManagedTimeIT.class) public static void doSetup() throws Exception { - Map props = Maps.newHashMapWithExpectedSize(1); - props.put(QueryServices.TRANSACTIONS_ENABLED, Boolean.toString(true)); - setUpTestDriver(new ReadOnlyProps(props.entrySet().iterator())); + Map serverProps = Maps.newHashMapWithExpectedSize(1); + serverProps.put("hbase.coprocessor.region.classes", CreateIndexRegionObserver.class.getName()); + Map clientProps = Maps.newHashMapWithExpectedSize(2); + clientProps.put(QueryServices.TRANSACTIONS_ENABLED, "true"); + setUpTestDriver(new ReadOnlyProps(serverProps.entrySet().iterator()), new ReadOnlyProps(clientProps.entrySet().iterator())); } - - @Parameters(name="localIndex = {0} , transactional = {1}") + + @Parameters(name="localIndex = {0} , transactional = {1}") public static Collection data() { return Arrays.asList(new Boolean[][] { - { false, false }, { false, true }, { true, false }, { true, true } - }); + { false, true }, { true, true } + }); } - + + @Test - public void testDropIfImmutableKeyValueColumn() throws Exception { - Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); + public void testCreateIndexDuringUpsertSelect() throws Exception { + Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); + props.setProperty(QueryServices.MUTATE_BATCH_SIZE_ATTRIB, Integer.toString(100)); + TABLE_NAME = fullTableName + "_testCreateIndexDuringUpsertSelect"; + String ddl ="CREATE TABLE " + TABLE_NAME + BaseTest.TEST_TABLE_SCHEMA + tableDDLOptions; + INDEX_DDL = "CREATE " + (localIndex ? "LOCAL" : "") + " INDEX IF NOT EXISTS " + indexName + " ON " + TABLE_NAME + + " (long_pk, varchar_pk)" + + " INCLUDE (long_col1, long_col2)"; + + Connection conn = DriverManager.getConnection(getUrl(), props); + try { + conn.setAutoCommit(false); + Statement stmt = conn.createStatement(); + stmt.execute(ddl); + + upsertRows(conn, TABLE_NAME, 220); + conn.commit(); + + // run the upsert select and also create an index + conn.setAutoCommit(true); + String upsertSelect = "UPSERT INTO " + TABLE_NAME + "(varchar_pk, char_pk, int_pk, long_pk, decimal_pk, date_pk) " + + "SELECT varchar_pk||'_upsert_select', char_pk, int_pk, long_pk, decimal_pk, date_pk FROM "+ TABLE_NAME; + conn.createStatement().execute(upsertSelect); + + ResultSet rs; + rs = conn.createStatement().executeQuery("SELECT /*+ NO_INDEX */ COUNT(*) FROM " + TABLE_NAME); + assertTrue(rs.next()); + assertEquals(440,rs.getInt(1)); + rs = conn.createStatement().executeQuery("SELECT COUNT(*) FROM " + TABLE_NAME); + assertTrue(rs.next()); + assertEquals(440,rs.getInt(1)); + } + finally { + conn.close(); + } + } + + // used to create an index while a batch of rows are being written + public static class CreateIndexRegionObserver extends SimpleRegionObserver { + @Override + public void postPut(ObserverContext c, + Put put, WALEdit edit, final Durability durability) + throws HBaseIOException { + String tableName = c.getEnvironment().getRegion().getRegionInfo() + .getTable().getNameAsString(); + if (tableName.equalsIgnoreCase(TABLE_NAME) + // create the index after the second batch of 1000 rows + && Bytes.startsWith(put.getRow(), Bytes.toBytes("varchar200_upsert_select"))) { + try { + Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); + try (Connection conn = DriverManager.getConnection(getUrl(), props)) { + conn.createStatement().execute(INDEX_DDL); + } + } catch (SQLException e) { + throw new DoNotRetryIOException(e); + } + } + } + } + + private static class UpsertRunnable implements Runnable { + private static final int NUM_ROWS_IN_BATCH = 10000; + private final String fullTableName; + + public UpsertRunnable(String fullTableName) { + this.fullTableName = fullTableName; + } + + public void run() { + Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); + try (Connection conn = DriverManager.getConnection(getUrl(), props)) { + while (true) { + // write a large batch of rows + boolean fistRowInBatch = true; + for (int i=0; i> futureList = Lists.newArrayListWithExpectedSize(numThreads); + for (int i =0; i future : futureList) { + future.cancel(true); + } + threadPool.shutdownNow(); + threadPool.awaitTermination(30, TimeUnit.SECONDS); + Thread.sleep(1000); + + ResultSet rs; + rs = conn.createStatement().executeQuery("SELECT /*+ NO_INDEX */ COUNT(*) FROM " + fullTableName); + assertTrue(rs.next()); + int dataTableRowCount = rs.getInt(1); + rs = conn.createStatement().executeQuery("SELECT COUNT(*) FROM " + fullIndexName); + assertTrue(rs.next()); + int indexTableRowCount = rs.getInt(1); + assertEquals("Data and Index table should have the same number of rows ", dataTableRowCount, indexTableRowCount); } } - } diff --git a/phoenix-core/src/main/java/org/apache/phoenix/execute/MutationState.java b/phoenix-core/src/main/java/org/apache/phoenix/execute/MutationState.java index a6fe98d1ea6..8caac5d5ce1 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/execute/MutationState.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/execute/MutationState.java @@ -135,7 +135,7 @@ public class MutationState implements SQLCloseable { private int numRows = 0; private int[] uncommittedStatementIndexes = EMPTY_STATEMENT_INDEX_ARRAY; private boolean isExternalTxContext = false; - private Map> txMutations; + private Map> txMutations = Collections.emptyMap(); private final MutationMetricQueue mutationMetricQueue; private ReadMetricQueue readMetricQueue; @@ -435,6 +435,59 @@ public long getUpdateCount() { return sizeOffset + numRows; } + private void joinMutationState(TableRef tableRef, Map srcRows, + Map> dstMutations) { + PTable table = tableRef.getTable(); + boolean isIndex = table.getType() == PTableType.INDEX; + boolean incrementRowCount = dstMutations == this.mutations; + Map existingRows = dstMutations.put(tableRef, srcRows); + if (existingRows != null) { // Rows for that table already exist + // Loop through new rows and replace existing with new + for (Map.Entry rowEntry : srcRows.entrySet()) { + // Replace existing row with new row + RowMutationState existingRowMutationState = existingRows.put(rowEntry.getKey(), rowEntry.getValue()); + if (existingRowMutationState != null) { + Map existingValues = existingRowMutationState.getColumnValues(); + if (existingValues != PRow.DELETE_MARKER) { + Map newRow = rowEntry.getValue().getColumnValues(); + // if new row is PRow.DELETE_MARKER, it means delete, and we don't need to merge it with existing row. + if (newRow != PRow.DELETE_MARKER) { + // Merge existing column values with new column values + existingRowMutationState.join(rowEntry.getValue()); + // Now that the existing row has been merged with the new row, replace it back + // again (since it was merged with the new one above). + existingRows.put(rowEntry.getKey(), existingRowMutationState); + } + } + } else { + if (incrementRowCount && !isIndex) { // Don't count index rows in row count + numRows++; + } + } + } + // Put the existing one back now that it's merged + dstMutations.put(tableRef, existingRows); + } else { + // Size new map at batch size as that's what it'll likely grow to. + Map newRows = Maps.newHashMapWithExpectedSize(connection.getMutateBatchSize()); + newRows.putAll(srcRows); + dstMutations.put(tableRef, newRows); + if (incrementRowCount && !isIndex) { + numRows += srcRows.size(); + } + } + } + + private void joinMutationState(Map> srcMutations, + Map> dstMutations) { + // Merge newMutation with this one, keeping state from newMutation for any overlaps + for (Map.Entry> entry : srcMutations.entrySet()) { + // Replace existing entries for the table with new entries + TableRef tableRef = entry.getKey(); + Map srcRows = entry.getValue(); + joinMutationState(tableRef, srcRows, dstMutations); + } + } /** * Combine a newer mutation with this one, where in the event of overlaps, the newer one will take precedence. * Combine any metrics collected for the newer mutation. @@ -453,48 +506,12 @@ public void join(MutationState newMutationState) { txAwares.addAll(newMutationState.txAwares); } this.sizeOffset += newMutationState.sizeOffset; - // Merge newMutation with this one, keeping state from newMutation for any overlaps - for (Map.Entry> entry : newMutationState.mutations.entrySet()) { - // Replace existing entries for the table with new entries - TableRef tableRef = entry.getKey(); - PTable table = tableRef.getTable(); - boolean isIndex = table.getType() == PTableType.INDEX; - Map existingRows = this.mutations.put(tableRef, entry.getValue()); - if (existingRows != null) { // Rows for that table already exist - // Loop through new rows and replace existing with new - for (Map.Entry rowEntry : entry.getValue().entrySet()) { - // Replace existing row with new row - RowMutationState existingRowMutationState = existingRows.put(rowEntry.getKey(), rowEntry.getValue()); - if (existingRowMutationState != null) { - Map existingValues = existingRowMutationState.getColumnValues(); - if (existingValues != PRow.DELETE_MARKER) { - Map newRow = rowEntry.getValue().getColumnValues(); - // if new row is PRow.DELETE_MARKER, it means delete, and we don't need to merge it with existing row. - if (newRow != PRow.DELETE_MARKER) { - // Merge existing column values with new column values - existingRowMutationState.join(rowEntry.getValue()); - // Now that the existing row has been merged with the new row, replace it back - // again (since it was merged with the new one above). - existingRows.put(rowEntry.getKey(), existingRowMutationState); - } - } - } else { - if (!isIndex) { // Don't count index rows in row count - numRows++; - } - } - } - // Put the existing one back now that it's merged - this.mutations.put(entry.getKey(), existingRows); - } else { - // Size new map at batch size as that's what it'll likely grow to. - Map newRows = Maps.newHashMapWithExpectedSize(connection.getMutateBatchSize()); - newRows.putAll(entry.getValue()); - this.mutations.put(tableRef, newRows); - if (!isIndex) { - numRows += entry.getValue().size(); - } + joinMutationState(newMutationState.mutations, this.mutations); + if (!newMutationState.txMutations.isEmpty()) { + if (txMutations.isEmpty()) { + txMutations = Maps.newHashMapWithExpectedSize(mutations.size()); } + joinMutationState(newMutationState.txMutations, this.txMutations); } mutationMetricQueue.combineMetricQueues(newMutationState.mutationMetricQueue); if (readMetricQueue == null) { @@ -915,6 +932,7 @@ private void send(Iterator tableRefIterator) throws SQLException { long startTime = System.currentTimeMillis(); child.addTimelineAnnotation("Attempt " + retryCount); hTable.batch(mutationList); + if (logger.isDebugEnabled()) logger.debug("Sent batch of " + numMutations + " for " + Bytes.toString(htableName)); child.stop(); child.stop(); shouldRetry = false; @@ -980,13 +998,13 @@ private void send(Iterator tableRefIterator) throws SQLException { // committed in the event of a failure. if (isTransactional) { addUncommittedStatementIndexes(valuesMap.values()); - if (txMutations == null) { + if (txMutations.isEmpty()) { txMutations = Maps.newHashMapWithExpectedSize(mutations.size()); } // Keep all mutations we've encountered until a commit or rollback. // This is not ideal, but there's not good way to get the values back // in the event that we need to replay the commit. - txMutations.put(tableRef, valuesMap); + joinMutationState(tableRef, valuesMap, txMutations); } // Remove batches as we process them if (sendAll) { @@ -1082,7 +1100,7 @@ private void resetState() { private void resetTransactionalState() { tx = null; txAwares.clear(); - txMutations = null; + txMutations = Collections.emptyMap(); uncommittedPhysicalNames.clear(); uncommittedStatementIndexes = EMPTY_STATEMENT_INDEX_ARRAY; } @@ -1187,9 +1205,7 @@ public void commit() throws SQLException { break; } retryCount++; - if (txMutations != null) { - mutations.putAll(txMutations); - } + mutations.putAll(txMutations); } while (true); } @@ -1214,6 +1230,7 @@ private boolean wasIndexAdded(Set txTableRefs) throws SQLException { if (result.getTable() == null) { throw new TableNotFoundException(dataTable.getSchemaName().getString(), dataTable.getTableName().getString()); } + tableRef.setTable(result.getTable()); if (!result.wasUpdated()) { if (logger.isInfoEnabled()) logger.info("No updates to " + dataTable.getName().getString() + " as of " + timestamp); continue; @@ -1223,7 +1240,7 @@ private boolean wasIndexAdded(Set txTableRefs) throws SQLException { // that an index was dropped and recreated with the same name but different // indexed/covered columns. addedIndexes = (!oldIndexes.equals(result.getTable().getIndexes())); - if (logger.isInfoEnabled()) logger.info((addedIndexes ? "Updates " : "No updates ") + "as of " + timestamp + " to " + dataTable.getName().getString() + " with indexes " + dataTable.getIndexes()); + if (logger.isInfoEnabled()) logger.info((addedIndexes ? "Updates " : "No updates ") + "as of " + timestamp + " to " + dataTable.getName().getString() + " with indexes " + tableRef.getTable().getIndexes()); } } if (logger.isInfoEnabled()) logger.info((addedIndexes ? "Updates " : "No updates ") + "to indexes as of " + getInitialWritePointer()); diff --git a/phoenix-core/src/test/java/org/apache/phoenix/query/BaseTest.java b/phoenix-core/src/test/java/org/apache/phoenix/query/BaseTest.java index 35bb8ce4ac1..951bfce2bec 100644 --- a/phoenix-core/src/test/java/org/apache/phoenix/query/BaseTest.java +++ b/phoenix-core/src/test/java/org/apache/phoenix/query/BaseTest.java @@ -17,6 +17,7 @@ */ package org.apache.phoenix.query; +import static org.apache.phoenix.query.BaseTest.setUpConfigForMiniCluster; import static org.apache.phoenix.util.PhoenixRuntime.CURRENT_SCN_ATTRIB; import static org.apache.phoenix.util.PhoenixRuntime.JDBC_PROTOCOL; import static org.apache.phoenix.util.PhoenixRuntime.JDBC_PROTOCOL_TERMINATOR; @@ -150,6 +151,7 @@ import org.apache.phoenix.hbase.index.balancer.IndexLoadBalancer; import org.apache.phoenix.hbase.index.master.IndexMasterObserver; import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData; +import org.apache.phoenix.jdbc.PhoenixDriver; import org.apache.phoenix.jdbc.PhoenixEmbeddedDriver; import org.apache.phoenix.jdbc.PhoenixEmbeddedDriver.ConnectionInfo; import org.apache.phoenix.jdbc.PhoenixTestDriver; @@ -200,7 +202,7 @@ public abstract class BaseTest { protected static final String TEST_TABLE_SCHEMA = "(" + " varchar_pk VARCHAR NOT NULL, " + - " char_pk CHAR(6) NOT NULL, " + + " char_pk CHAR(10) NOT NULL, " + " int_pk INTEGER NOT NULL, "+ " long_pk BIGINT NOT NULL, " + " decimal_pk DECIMAL(31, 10) NOT NULL, " + @@ -1805,77 +1807,44 @@ public static void assertValuesEqualsResultSet(ResultSet rs, List> public HBaseTestingUtility getUtility() { return utility; } + + public static void upsertRows(Connection conn, String fullTableName, int numRows) throws SQLException { + for (int i=1; i<=numRows; ++i) { + upsertRow(conn, fullTableName, i, false); + } + } + + public static void upsertRow(Connection conn, String fullTableName, int index, boolean firstRowInBatch) throws SQLException { + String upsert = "UPSERT INTO " + fullTableName + + " VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"; + PreparedStatement stmt = conn.prepareStatement(upsert); + stmt.setString(1, firstRowInBatch ? "firstRowInBatch_" : "" + "varchar"+index); + stmt.setString(2, "char"+index); + stmt.setInt(3, index); + stmt.setLong(4, index); + stmt.setBigDecimal(5, new BigDecimal(index)); + Date date = DateUtil.parseDate("2015-01-01 00:00:00"); + stmt.setDate(6, date); + stmt.setString(7, "varchar_a"); + stmt.setString(8, "chara"); + stmt.setInt(9, index+1); + stmt.setLong(10, index+1); + stmt.setBigDecimal(11, new BigDecimal(index+1)); + stmt.setDate(12, date); + stmt.setString(13, "varchar_b"); + stmt.setString(14, "charb"); + stmt.setInt(15, index+2); + stmt.setLong(16, index+2); + stmt.setBigDecimal(17, new BigDecimal(index+2)); + stmt.setDate(18, date); + stmt.executeUpdate(); + } // Populate the test table with data. public static void populateTestTable(String fullTableName) throws SQLException { Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); try (Connection conn = DriverManager.getConnection(getUrl(), props)) { - String upsert = "UPSERT INTO " + fullTableName - + " VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"; - PreparedStatement stmt = conn.prepareStatement(upsert); - stmt.setString(1, "varchar1"); - stmt.setString(2, "char1"); - stmt.setInt(3, 1); - stmt.setLong(4, 1L); - stmt.setBigDecimal(5, new BigDecimal(1.0)); - Date date = DateUtil.parseDate("2015-01-01 00:00:00"); - stmt.setDate(6, date); - stmt.setString(7, "varchar_a"); - stmt.setString(8, "chara"); - stmt.setInt(9, 2); - stmt.setLong(10, 2L); - stmt.setBigDecimal(11, new BigDecimal(2.0)); - stmt.setDate(12, date); - stmt.setString(13, "varchar_b"); - stmt.setString(14, "charb"); - stmt.setInt(15, 3); - stmt.setLong(16, 3L); - stmt.setBigDecimal(17, new BigDecimal(3.0)); - stmt.setDate(18, date); - stmt.executeUpdate(); - - stmt.setString(1, "varchar2"); - stmt.setString(2, "char2"); - stmt.setInt(3, 2); - stmt.setLong(4, 2L); - stmt.setBigDecimal(5, new BigDecimal(2.0)); - date = DateUtil.parseDate("2015-01-02 00:00:00"); - stmt.setDate(6, date); - stmt.setString(7, "varchar_a"); - stmt.setString(8, "chara"); - stmt.setInt(9, 3); - stmt.setLong(10, 3L); - stmt.setBigDecimal(11, new BigDecimal(3.0)); - stmt.setDate(12, date); - stmt.setString(13, "varchar_b"); - stmt.setString(14, "charb"); - stmt.setInt(15, 4); - stmt.setLong(16, 4L); - stmt.setBigDecimal(17, new BigDecimal(4.0)); - stmt.setDate(18, date); - stmt.executeUpdate(); - - stmt.setString(1, "varchar3"); - stmt.setString(2, "char3"); - stmt.setInt(3, 3); - stmt.setLong(4, 3L); - stmt.setBigDecimal(5, new BigDecimal(3.0)); - date = DateUtil.parseDate("2015-01-03 00:00:00"); - stmt.setDate(6, date); - stmt.setString(7, "varchar_a"); - stmt.setString(8, "chara"); - stmt.setInt(9, 4); - stmt.setLong(10, 4L); - stmt.setBigDecimal(11, new BigDecimal(4.0)); - stmt.setDate(12, date); - stmt.setString(13, "varchar_b"); - stmt.setString(14, "charb"); - stmt.setInt(15, 5); - stmt.setLong(16, 5L); - stmt.setBigDecimal(17, new BigDecimal(5.0)); - stmt.setDate(18, date); - stmt.executeUpdate(); - + upsertRows(conn, fullTableName, 3); conn.commit(); } } From 842a374187bcf1b199b9832de4b0ec68d62291db Mon Sep 17 00:00:00 2001 From: Ankit Singhal Date: Wed, 20 Jan 2016 00:34:43 +0530 Subject: [PATCH 23/47] Delete stats rows at server side for 4.7 upgrade --- .../coprocessor/MetaDataRegionObserver.java | 141 +++++++++++++++--- 1 file changed, 122 insertions(+), 19 deletions(-) diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataRegionObserver.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataRegionObserver.java index c40e3cd435c..c133cbef99e 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataRegionObserver.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataRegionObserver.java @@ -18,10 +18,6 @@ package org.apache.phoenix.coprocessor; import java.io.IOException; - -import org.apache.log4j.Level; -import org.apache.log4j.Logger; -import org.apache.phoenix.jdbc.PhoenixDriver; import java.sql.DriverManager; import java.sql.SQLException; import java.util.ArrayList; @@ -36,29 +32,46 @@ import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CoprocessorEnvironment; import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Delete; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.HTableInterface; +import org.apache.hadoop.hbase.client.Mutation; +import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.ResultScanner; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver; import org.apache.hadoop.hbase.coprocessor.ObserverContext; import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment; -import org.apache.hadoop.hbase.filter.SingleColumnValueFilter; import org.apache.hadoop.hbase.filter.CompareFilter; +import org.apache.hadoop.hbase.filter.SingleColumnValueFilter; import org.apache.hadoop.hbase.regionserver.RegionScanner; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.log4j.Level; +import org.apache.log4j.Logger; import org.apache.phoenix.cache.GlobalCache; import org.apache.phoenix.jdbc.PhoenixConnection; import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData; +import org.apache.phoenix.jdbc.PhoenixDriver; +import org.apache.phoenix.query.QueryConstants; import org.apache.phoenix.query.QueryServices; import org.apache.phoenix.query.QueryServicesOptions; import org.apache.phoenix.schema.MetaDataClient; import org.apache.phoenix.schema.PIndexState; -import org.apache.phoenix.schema.types.PLong; import org.apache.phoenix.schema.PTable; import org.apache.phoenix.schema.TableRef; +import org.apache.phoenix.schema.types.PLong; +import org.apache.phoenix.schema.types.PVarchar; +import org.apache.phoenix.util.ByteUtil; +import org.apache.phoenix.util.KeyValueUtil; import org.apache.phoenix.util.MetaDataUtil; import org.apache.phoenix.util.PhoenixRuntime; import org.apache.phoenix.util.SchemaUtil; +import com.google.common.collect.Lists; + /** * Coprocessor for metadata related operations. This coprocessor would only be registered @@ -69,6 +82,8 @@ public class MetaDataRegionObserver extends BaseRegionObserver { protected ScheduledThreadPoolExecutor executor = new ScheduledThreadPoolExecutor(1); private boolean enableRebuildIndex = QueryServicesOptions.DEFAULT_INDEX_FAILURE_HANDLING_REBUILD; private long rebuildIndexTimeInterval = QueryServicesOptions.DEFAULT_INDEX_FAILURE_HANDLING_REBUILD_INTERVAL; + public static final byte[] UPGRADE_TO_4_7_COLUMN_NAME = Bytes.toBytes("UPGRADE_TO_4_7"); + public static final String UPGRADE_TO_4_7_COLUMN_VALUE = "TRUNCATED_4_7"; @Override public void preClose(final ObserverContext c, @@ -96,10 +111,109 @@ public void start(CoprocessorEnvironment env) throws IOException { rebuildIndexTimeInterval = env.getConfiguration().getLong(QueryServices.INDEX_FAILURE_HANDLING_REBUILD_INTERVAL_ATTRIB, QueryServicesOptions.DEFAULT_INDEX_FAILURE_HANDLING_REBUILD_INTERVAL); } - + + private static String getJdbcUrl(RegionCoprocessorEnvironment env) { + String zkQuorum = env.getConfiguration().get(HConstants.ZOOKEEPER_QUORUM); + String zkClientPort = env.getConfiguration().get(HConstants.ZOOKEEPER_CLIENT_PORT, + Integer.toString(HConstants.DEFAULT_ZOOKEPER_CLIENT_PORT)); + String zkParentNode = env.getConfiguration().get(HConstants.ZOOKEEPER_ZNODE_PARENT, + HConstants.DEFAULT_ZOOKEEPER_ZNODE_PARENT); + return PhoenixRuntime.JDBC_PROTOCOL + PhoenixRuntime.JDBC_PROTOCOL_SEPARATOR + zkQuorum + + PhoenixRuntime.JDBC_PROTOCOL_SEPARATOR + zkClientPort + + PhoenixRuntime.JDBC_PROTOCOL_SEPARATOR + zkParentNode; + } @Override public void postOpen(ObserverContext e) { + final RegionCoprocessorEnvironment env = e.getEnvironment(); + + Runnable r = new Runnable() { + @Override + public void run() { + HTableInterface metaTable = null; + HTableInterface statsTable = null; + try { + Thread.sleep(1000); + LOG.info("Stats will be deleted for upgrade 4.7 requirement!!"); + metaTable = env.getTable(TableName.valueOf(PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME)); + List columnCells = metaTable + .get(new Get(SchemaUtil.getTableKey(null, PhoenixDatabaseMetaData.SYSTEM_SCHEMA_NAME, + PhoenixDatabaseMetaData.SYSTEM_CATALOG_TABLE))) + .getColumnCells(PhoenixDatabaseMetaData.TABLE_FAMILY_BYTES, + QueryConstants.EMPTY_COLUMN_BYTES); + if (!columnCells.isEmpty() + && columnCells.get(0).getTimestamp() < MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_7_0) { + + statsTable = env.getTable(TableName.valueOf(PhoenixDatabaseMetaData.SYSTEM_STATS_NAME)); + byte[] statsTableKey = SchemaUtil.getTableKey(null, PhoenixDatabaseMetaData.SYSTEM_SCHEMA_NAME, + PhoenixDatabaseMetaData.SYSTEM_STATS_TABLE); + KeyValue upgradeKV = KeyValueUtil.newKeyValue(statsTableKey, + PhoenixDatabaseMetaData.TABLE_FAMILY_BYTES, UPGRADE_TO_4_7_COLUMN_NAME, + MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_7_0 - 1, + PVarchar.INSTANCE.toBytes(UPGRADE_TO_4_7_COLUMN_VALUE)); + Put upgradePut = new Put(statsTableKey); + upgradePut.add(upgradeKV); + + // check for null in UPGRADE_TO_4_7_COLUMN_NAME in checkAndPut so that only single client + // drop the rows of SYSTEM.STATS + if (metaTable.checkAndPut(statsTableKey, PhoenixDatabaseMetaData.TABLE_FAMILY_BYTES, + UPGRADE_TO_4_7_COLUMN_NAME, null, upgradePut)) { + List mutations = Lists.newArrayListWithExpectedSize(1000); + Scan scan = new Scan(); + scan.setRaw(true); + scan.setMaxVersions(); + ResultScanner statsScanner = statsTable.getScanner(scan); + Result r; + mutations.clear(); + int count = 0; + while ((r = statsScanner.next()) != null) { + Delete delete = null; + for (KeyValue keyValue : r.raw()) { + if (KeyValue.Type.codeToType(keyValue.getType()) == KeyValue.Type.Put) { + if (delete == null) { + delete = new Delete(keyValue.getRow()); + } + KeyValue deleteKeyValue = new KeyValue(keyValue.getRowArray(), + keyValue.getRowOffset(), keyValue.getRowLength(), + keyValue.getFamilyArray(), keyValue.getFamilyOffset(), + keyValue.getFamilyLength(), keyValue.getQualifierArray(), + keyValue.getQualifierOffset(), keyValue.getQualifierLength(), + keyValue.getTimestamp(), KeyValue.Type.Delete, + ByteUtil.EMPTY_BYTE_ARRAY, 0, 0); + delete.addDeleteMarker(deleteKeyValue); + } + } + if (delete != null) { + mutations.add(delete); + if (count > 1000) { + statsTable.batch(mutations); + mutations.clear(); + count = 0; + } + count++; + } + } + if (!mutations.isEmpty()) { + statsTable.batch(mutations); + } + } + } + } catch (Exception e) { + LOG.warn("Exception while deleting stats..", e); + } finally { + try { + if (metaTable != null) { + metaTable.close(); + } + if (statsTable != null) { + statsTable.close(); + } + } catch (IOException e) {} + } + } + }; + (new Thread(r)).start(); + if (!enableRebuildIndex) { LOG.info("Failure Index Rebuild is skipped by configuration."); return; @@ -134,17 +248,6 @@ public BuildIndexScheduleTask(RegionCoprocessorEnvironment env) { this.env = env; } - private String getJdbcUrl() { - String zkQuorum = this.env.getConfiguration().get(HConstants.ZOOKEEPER_QUORUM); - String zkClientPort = this.env.getConfiguration().get(HConstants.ZOOKEEPER_CLIENT_PORT, - Integer.toString(HConstants.DEFAULT_ZOOKEPER_CLIENT_PORT)); - String zkParentNode = this.env.getConfiguration().get(HConstants.ZOOKEEPER_ZNODE_PARENT, - HConstants.DEFAULT_ZOOKEEPER_ZNODE_PARENT); - return PhoenixRuntime.JDBC_PROTOCOL + PhoenixRuntime.JDBC_PROTOCOL_SEPARATOR + zkQuorum - + PhoenixRuntime.JDBC_PROTOCOL_SEPARATOR + zkClientPort - + PhoenixRuntime.JDBC_PROTOCOL_SEPARATOR + zkParentNode; - } - @Override public void run() { RegionScanner scanner = null; @@ -219,7 +322,7 @@ public void run() { } if (conn == null) { - conn = DriverManager.getConnection(getJdbcUrl()).unwrap(PhoenixConnection.class); + conn = DriverManager.getConnection(getJdbcUrl(env)).unwrap(PhoenixConnection.class); } String dataTableFullName = SchemaUtil.getTableName(schemaName, dataTable); From 94df934751205e5e8901055ddb752df4c894d493 Mon Sep 17 00:00:00 2001 From: Ankit Singhal Date: Wed, 20 Jan 2016 01:13:41 +0530 Subject: [PATCH 24/47] Reducing delete batch size from 1000 to 10 for stats table as each row could be too big --- .../org/apache/phoenix/coprocessor/MetaDataRegionObserver.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataRegionObserver.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataRegionObserver.java index c133cbef99e..64d5f3cef34 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataRegionObserver.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataRegionObserver.java @@ -185,7 +185,7 @@ public void run() { } if (delete != null) { mutations.add(delete); - if (count > 1000) { + if (count > 10) { statsTable.batch(mutations); mutations.clear(); count = 0; From 1717f12d7c6e9b1ee8ddffb55cea929d6d8543f1 Mon Sep 17 00:00:00 2001 From: James Taylor Date: Tue, 19 Jan 2016 13:16:50 -0800 Subject: [PATCH 25/47] PHOENIX-2478 Rows committed in transaction overlapping index creation are not populated --- .../main/java/org/apache/phoenix/execute/MutationState.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/phoenix-core/src/main/java/org/apache/phoenix/execute/MutationState.java b/phoenix-core/src/main/java/org/apache/phoenix/execute/MutationState.java index 8caac5d5ce1..d3b36ec2551 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/execute/MutationState.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/execute/MutationState.java @@ -1231,10 +1231,6 @@ private boolean wasIndexAdded(Set txTableRefs) throws SQLException { throw new TableNotFoundException(dataTable.getSchemaName().getString(), dataTable.getTableName().getString()); } tableRef.setTable(result.getTable()); - if (!result.wasUpdated()) { - if (logger.isInfoEnabled()) logger.info("No updates to " + dataTable.getName().getString() + " as of " + timestamp); - continue; - } if (!addedIndexes) { // TODO: in theory we should do a deep equals check here, as it's possible // that an index was dropped and recreated with the same name but different From f572fa630a5bdfe5f1eab074838a82344d38063d Mon Sep 17 00:00:00 2001 From: James Taylor Date: Tue, 19 Jan 2016 20:23:51 -0800 Subject: [PATCH 26/47] PHOENIX-2478 Rows committed in transaction overlapping index creation are not populated --- .../end2end/index/ImmutableIndexIT.java | 6 +- .../end2end/index/IndexExpressionIT.java | 16 +-- .../phoenix/compile/PostIndexDDLCompiler.java | 2 +- .../phoenix/exception/SQLExceptionCode.java | 108 +++++++++--------- .../apache/phoenix/execute/MutationState.java | 39 ++++--- .../apache/phoenix/jdbc/PhoenixStatement.java | 2 +- .../apache/phoenix/schema/MetaDataClient.java | 2 +- .../org/apache/phoenix/schema/TableRef.java | 8 +- .../phoenix/compile/QueryCompilerTest.java | 2 +- 9 files changed, 98 insertions(+), 87 deletions(-) diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/ImmutableIndexIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/ImmutableIndexIT.java index c4ecfbb627f..71713826d96 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/ImmutableIndexIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/ImmutableIndexIT.java @@ -219,7 +219,7 @@ public void testCreateIndexWhileUpsertingData() throws Exception { futureList.add(threadPool.submit(new UpsertRunnable(fullTableName))); } // upsert some rows before creating the index - Thread.sleep(5000); + Thread.sleep(500); // create the index try (Connection conn2 = DriverManager.getConnection(getUrl(), props)) { @@ -230,14 +230,14 @@ public void testCreateIndexWhileUpsertingData() throws Exception { } // upsert some rows after creating the index - Thread.sleep(1000); + Thread.sleep(100); // cancel the running threads for (Future future : futureList) { future.cancel(true); } threadPool.shutdownNow(); threadPool.awaitTermination(30, TimeUnit.SECONDS); - Thread.sleep(1000); + Thread.sleep(100); ResultSet rs; rs = conn.createStatement().executeQuery("SELECT /*+ NO_INDEX */ COUNT(*) FROM " + fullTableName); diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexExpressionIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexExpressionIT.java index 7da0d85808d..7be8d41ecc5 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexExpressionIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexExpressionIT.java @@ -95,7 +95,7 @@ private void insertRow(PreparedStatement stmt, int i) throws SQLException { private void verifyResult(ResultSet rs, int i) throws SQLException { assertTrue(rs.next()); - assertEquals("VARCHAR" + String.valueOf(i) + "_" + StringUtils.rightPad("CHAR" + String.valueOf(i), 6, ' ') + assertEquals("VARCHAR" + String.valueOf(i) + "_" + StringUtils.rightPad("CHAR" + String.valueOf(i), 10, ' ') + "_A.VARCHAR" + String.valueOf(i) + "_" + StringUtils.rightPad("B.CHAR" + String.valueOf(i), 10, ' '), rs.getString(1)); assertEquals(i * 3, rs.getInt(2)); @@ -141,7 +141,7 @@ protected void helpTestCreateAndUpdate(boolean mutable, boolean localIndex) thro // DECIMAL in the index (which is not fixed width) + " AND date_pk+1=? AND date1+1=? AND date2+1=?"; stmt = conn.prepareStatement(whereSql); - stmt.setString(1, "VARCHAR1_CHAR1 _A.VARCHAR1_B.CHAR1 "); + stmt.setString(1, "VARCHAR1_CHAR1 _A.VARCHAR1_B.CHAR1 "); stmt.setInt(2, 3); Date date = DateUtil.parseDate("2015-01-02 00:00:00"); stmt.setDate(3, date); @@ -153,8 +153,8 @@ protected void helpTestCreateAndUpdate(boolean mutable, boolean localIndex) thro assertEquals( localIndex ? "CLIENT PARALLEL 1-WAY RANGE SCAN OVER _LOCAL_IDX_INDEX_TEST." + dataTableName - + " [-32768,'VARCHAR1_CHAR1 _A.VARCHAR1_B.CHAR1 ',3,'2015-01-02 00:00:00.000',1,420,156,800,000,1,420,156,800,000]\nCLIENT MERGE SORT" - : "CLIENT PARALLEL 1-WAY RANGE SCAN OVER INDEX_TEST.IDX ['VARCHAR1_CHAR1 _A.VARCHAR1_B.CHAR1 ',3,'2015-01-02 00:00:00.000',1,420,156,800,000,1,420,156,800,000]", + + " [-32768,'VARCHAR1_CHAR1 _A.VARCHAR1_B.CHAR1 ',3,'2015-01-02 00:00:00.000',1,420,156,800,000,1,420,156,800,000]\nCLIENT MERGE SORT" + : "CLIENT PARALLEL 1-WAY RANGE SCAN OVER INDEX_TEST.IDX ['VARCHAR1_CHAR1 _A.VARCHAR1_B.CHAR1 ',3,'2015-01-02 00:00:00.000',1,420,156,800,000,1,420,156,800,000]", QueryUtil.getExplainPlan(rs)); // verify that the correct results are returned @@ -254,20 +254,20 @@ protected void helpTestUpdate(boolean localIndex) throws Exception { + fullDataTableName; ResultSet rs = conn.createStatement().executeQuery("SELECT /*+ NO_INDEX */ " + selectSql); assertTrue(rs.next()); - assertEquals("VARCHAR1_CHAR1 _A.VARCHAR_UPDATED_B.CHAR1 ", rs.getString(1)); + assertEquals("VARCHAR1_CHAR1 _A.VARCHAR_UPDATED_B.CHAR1 ", rs.getString(1)); assertEquals(101, rs.getLong(2)); assertTrue(rs.next()); - assertEquals("VARCHAR2_CHAR2 _A.VARCHAR2_B.CHAR2 ", rs.getString(1)); + assertEquals("VARCHAR2_CHAR2 _A.VARCHAR2_B.CHAR2 ", rs.getString(1)); assertEquals(2, rs.getLong(2)); assertFalse(rs.next()); // verify that the rows in the index table are also updated rs = conn.createStatement().executeQuery("SELECT " + selectSql); assertTrue(rs.next()); - assertEquals("VARCHAR1_CHAR1 _A.VARCHAR_UPDATED_B.CHAR1 ", rs.getString(1)); + assertEquals("VARCHAR1_CHAR1 _A.VARCHAR_UPDATED_B.CHAR1 ", rs.getString(1)); assertEquals(101, rs.getLong(2)); assertTrue(rs.next()); - assertEquals("VARCHAR2_CHAR2 _A.VARCHAR2_B.CHAR2 ", rs.getString(1)); + assertEquals("VARCHAR2_CHAR2 _A.VARCHAR2_B.CHAR2 ", rs.getString(1)); assertEquals(2, rs.getLong(2)); assertFalse(rs.next()); conn.createStatement().execute("DROP INDEX IDX ON " + fullDataTableName); diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/PostIndexDDLCompiler.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/PostIndexDDLCompiler.java index bb0b59516e1..1a667ae1f20 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/compile/PostIndexDDLCompiler.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/PostIndexDDLCompiler.java @@ -120,7 +120,7 @@ public MutationPlan compile(final PTable indexTable) throws SQLException { DelegateMutationPlan delegate = new DelegateMutationPlan(statement.compileMutation(updateStmtStr.toString())) { @Override public MutationState execute() throws SQLException { - connection.getMutationState().commitWriteFence(dataTable); + connection.getMutationState().commitDDLFence(dataTable); return super.execute(); } }; diff --git a/phoenix-core/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java b/phoenix-core/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java index 9767cbea2ba..b1d8e7db6e9 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java @@ -131,53 +131,53 @@ public SQLException newException(SQLExceptionInfo info) { PRIMARY_KEY_MISSING(509, "42888", "The table does not have a primary key."), PRIMARY_KEY_ALREADY_EXISTS(510, "42889", "The table already has a primary key."), ORDER_BY_NOT_IN_SELECT_DISTINCT(511, "42890", "All ORDER BY expressions must appear in SELECT DISTINCT:"), - INVALID_PRIMARY_KEY_CONSTRAINT(512, "42891", "Invalid column reference in primary key constraint"), - ARRAY_NOT_ALLOWED_IN_PRIMARY_KEY(513, "42892", "Array type not allowed as primary key constraint"), + INVALID_PRIMARY_KEY_CONSTRAINT(512, "42891", "Invalid column reference in primary key constraint."), + ARRAY_NOT_ALLOWED_IN_PRIMARY_KEY(513, "42892", "Array type not allowed as primary key constraint."), COLUMN_EXIST_IN_DEF(514, "42892", "A duplicate column name was detected in the object definition or ALTER TABLE statement.", new Factory() { @Override public SQLException newException(SQLExceptionInfo info) { return new ColumnAlreadyExistsException(info.getSchemaName(), info.getTableName(), info.getColumnName()); } }), - ORDER_BY_ARRAY_NOT_SUPPORTED(515, "42893", "ORDER BY of an array type is not allowed"), - NON_EQUALITY_ARRAY_COMPARISON(516, "42894", "Array types may only be compared using = or !="), - INVALID_NOT_NULL_CONSTRAINT(517, "42895", "Invalid not null constraint on non primary key column"), + ORDER_BY_ARRAY_NOT_SUPPORTED(515, "42893", "ORDER BY of an array type is not allowed."), + NON_EQUALITY_ARRAY_COMPARISON(516, "42894", "Array types may only be compared using = or !=."), + INVALID_NOT_NULL_CONSTRAINT(517, "42895", "Invalid not null constraint on non primary key column."), /** * Invalid Transaction State (errorcode 05, sqlstate 25) */ READ_ONLY_CONNECTION(518,"25502","Mutations are not permitted for a read-only connection."), - VARBINARY_ARRAY_NOT_SUPPORTED(519, "42896", "VARBINARY ARRAY is not supported"), + VARBINARY_ARRAY_NOT_SUPPORTED(519, "42896", "VARBINARY ARRAY is not supported."), /** * Expression Index exceptions. */ - AGGREGATE_EXPRESSION_NOT_ALLOWED_IN_INDEX(520, "42897", "Aggregate expression not allowed in an index"), - NON_DETERMINISTIC_EXPRESSION_NOT_ALLOWED_IN_INDEX(521, "42898", "Non-deterministic expression not allowed in an index"), - STATELESS_EXPRESSION_NOT_ALLOWED_IN_INDEX(522, "42899", "Stateless expression not allowed in an index"), + AGGREGATE_EXPRESSION_NOT_ALLOWED_IN_INDEX(520, "42897", "Aggregate expression not allowed in an index."), + NON_DETERMINISTIC_EXPRESSION_NOT_ALLOWED_IN_INDEX(521, "42898", "Non-deterministic expression not allowed in an index."), + STATELESS_EXPRESSION_NOT_ALLOWED_IN_INDEX(522, "42899", "Stateless expression not allowed in an index."), /** * Transaction exceptions. */ - TRANSACTION_CONFLICT_EXCEPTION(523, "42900", "Transaction aborted due to conflict with other mutations"), - TRANSACTION_EXCEPTION(524, "42901", "Transaction aborted due to error"), + TRANSACTION_CONFLICT_EXCEPTION(523, "42900", "Transaction aborted due to conflict with other mutations."), + TRANSACTION_EXCEPTION(524, "42901", "Transaction aborted due to error."), /** * Union All related errors */ - SELECT_COLUMN_NUM_IN_UNIONALL_DIFFS(525, "42902", "SELECT column number differs in a Union All query is not allowed"), - SELECT_COLUMN_TYPE_IN_UNIONALL_DIFFS(526, "42903", "SELECT column types differ in a Union All query is not allowed"), + SELECT_COLUMN_NUM_IN_UNIONALL_DIFFS(525, "42902", "SELECT column number differs in a Union All query is not allowed."), + SELECT_COLUMN_TYPE_IN_UNIONALL_DIFFS(526, "42903", "SELECT column types differ in a Union All query is not allowed."), /** * Row timestamp column related errors */ - ROWTIMESTAMP_ONE_PK_COL_ONLY(527, "42904", "Only one column that is part of the primary key can be declared as a ROW_TIMESTAMP"), - ROWTIMESTAMP_PK_COL_ONLY(528, "42905", "Only columns part of the primary key can be declared as a ROW_TIMESTAMP"), - ROWTIMESTAMP_CREATE_ONLY(529, "42906", "A column can be added as ROW_TIMESTAMP only in CREATE TABLE"), - ROWTIMESTAMP_COL_INVALID_TYPE(530, "42907", "A column can be added as ROW_TIMESTAMP only if it is of type DATE, BIGINT, TIME OR TIMESTAMP"), - ROWTIMESTAMP_NOT_ALLOWED_ON_VIEW(531, "42908", "Declaring a column as row_timestamp is not allowed for views"), - INVALID_SCN(532, "42909", "Value of SCN cannot be less than zero"), + ROWTIMESTAMP_ONE_PK_COL_ONLY(527, "42904", "Only one column that is part of the primary key can be declared as a ROW_TIMESTAMP."), + ROWTIMESTAMP_PK_COL_ONLY(528, "42905", "Only columns part of the primary key can be declared as a ROW_TIMESTAMP."), + ROWTIMESTAMP_CREATE_ONLY(529, "42906", "A column can be added as ROW_TIMESTAMP only in CREATE TABLE."), + ROWTIMESTAMP_COL_INVALID_TYPE(530, "42907", "A column can be added as ROW_TIMESTAMP only if it is of type DATE, BIGINT, TIME OR TIMESTAMP."), + ROWTIMESTAMP_NOT_ALLOWED_ON_VIEW(531, "42908", "Declaring a column as row_timestamp is not allowed for views."), + INVALID_SCN(532, "42909", "Value of SCN cannot be less than zero."), /** * HBase and Phoenix specific implementation defined sub-classes. * Column family related exceptions. @@ -241,10 +241,10 @@ public SQLException newException(SQLExceptionInfo info) { INVALID_INDEX_STATE_TRANSITION(1028, "42Y87", "Invalid index state transition."), INVALID_MUTABLE_INDEX_CONFIG(1029, "42Y88", "Mutable secondary indexes must have the " + IndexManagementUtil.WAL_EDIT_CODEC_CLASS_KEY + " property set to " - + IndexManagementUtil.INDEX_WAL_EDIT_CODEC_CLASS_NAME + " in the hbase-sites.xml of every region server"), + + IndexManagementUtil.INDEX_WAL_EDIT_CODEC_CLASS_NAME + " in the hbase-sites.xml of every region server."), - CANNOT_CREATE_TENANT_SPECIFIC_TABLE(1030, "42Y89", "Cannot create table for tenant-specific connection"), + CANNOT_CREATE_TENANT_SPECIFIC_TABLE(1030, "42Y89", "Cannot create table for tenant-specific connection."), DEFAULT_COLUMN_FAMILY_ONLY_ON_CREATE_TABLE(1034, "42Y93", "Default column family may only be specified when creating a table."), INSUFFICIENT_MULTI_TENANT_COLUMNS(1040, "42Y96", "A MULTI_TENANT table must have two or more PK columns with the first column being NOT NULL."), TENANTID_IS_OF_WRONG_TYPE(1041, "42Y97", "The TenantId could not be converted to correct format for this table."), @@ -254,33 +254,33 @@ public SQLException newException(SQLExceptionInfo info) { NO_LOCAL_INDEX_ON_TABLE_WITH_IMMUTABLE_ROWS(1048,"43A05","Local indexes aren't allowed on tables with immutable rows."), COLUMN_FAMILY_NOT_ALLOWED_TABLE_PROPERTY(1049, "43A06", "Column family not allowed for table properties."), COLUMN_FAMILY_NOT_ALLOWED_FOR_TTL(1050, "43A07", "Setting TTL for a column family not supported. You can only have TTL for the entire table."), - CANNOT_ALTER_PROPERTY(1051, "43A08", "Property can be specified or changed only when creating a table"), - CANNOT_SET_PROPERTY_FOR_COLUMN_NOT_ADDED(1052, "43A09", "Property cannot be specified for a column family that is not being added or modified"), - CANNOT_SET_TABLE_PROPERTY_ADD_COLUMN(1053, "43A10", "Table level property cannot be set when adding a column"), + CANNOT_ALTER_PROPERTY(1051, "43A08", "Property can be specified or changed only when creating a table."), + CANNOT_SET_PROPERTY_FOR_COLUMN_NOT_ADDED(1052, "43A09", "Property cannot be specified for a column family that is not being added or modified."), + CANNOT_SET_TABLE_PROPERTY_ADD_COLUMN(1053, "43A10", "Table level property cannot be set when adding a column."), NO_LOCAL_INDEXES(1054, "43A11", "Local secondary indexes are not supported for HBase versions " + MetaDataUtil.decodeHBaseVersionAsString(PhoenixDatabaseMetaData.MIN_LOCAL_SI_VERSION_DISALLOW) + " through " + MetaDataUtil.decodeHBaseVersionAsString(PhoenixDatabaseMetaData.MAX_LOCAL_SI_VERSION_DISALLOW) + " inclusive."), UNALLOWED_LOCAL_INDEXES(1055, "43A12", "Local secondary indexes are configured to not be allowed."), - DESC_VARBINARY_NOT_SUPPORTED(1056, "43A13", "Descending VARBINARY columns not supported"), + DESC_VARBINARY_NOT_SUPPORTED(1056, "43A13", "Descending VARBINARY columns not supported."), - DEFAULT_COLUMN_FAMILY_ON_SHARED_TABLE(1069, "43A13", "Default column family not allowed on VIEW or shared INDEX"), - ONLY_TABLE_MAY_BE_DECLARED_TRANSACTIONAL(1070, "44A01", "Only tables may be declared as transactional"), - TX_MAY_NOT_SWITCH_TO_NON_TX(1071, "44A02", "A transactional table may not be switched to non transactional"), - STORE_NULLS_MUST_BE_TRUE_FOR_TRANSACTIONAL(1072, "44A03", "Store nulls must be true when a table is transactional"), - CANNOT_START_TRANSACTION_WITH_SCN_SET(1073, "44A04", "Cannot start a transaction on a connection with SCN set"), - TX_MAX_VERSIONS_MUST_BE_GREATER_THAN_ONE(1074, "44A05", "A transactional table must define VERSION of greater than one"), - CANNOT_SPECIFY_SCN_FOR_TXN_TABLE(1075, "44A06", "Cannot use a connection with SCN set for a transactional table"), - NULL_TRANSACTION_CONTEXT(1076, "44A07", "No Transaction Context available"), + DEFAULT_COLUMN_FAMILY_ON_SHARED_TABLE(1069, "43A13", "Default column family not allowed on VIEW or shared INDEX."), + ONLY_TABLE_MAY_BE_DECLARED_TRANSACTIONAL(1070, "44A01", "Only tables may be declared as transactional."), + TX_MAY_NOT_SWITCH_TO_NON_TX(1071, "44A02", "A transactional table may not be switched to non transactional."), + STORE_NULLS_MUST_BE_TRUE_FOR_TRANSACTIONAL(1072, "44A03", "Store nulls must be true when a table is transactional."), + CANNOT_START_TRANSACTION_WITH_SCN_SET(1073, "44A04", "Cannot start a transaction on a connection with SCN set."), + TX_MAX_VERSIONS_MUST_BE_GREATER_THAN_ONE(1074, "44A05", "A transactional table must define VERSION of greater than one."), + CANNOT_SPECIFY_SCN_FOR_TXN_TABLE(1075, "44A06", "Cannot use a connection with SCN set for a transactional table."), + NULL_TRANSACTION_CONTEXT(1076, "44A07", "No Transaction Context available."), TRANSACTION_FAILED(1077, "44A08", "Transaction Failure "), - CANNOT_CREATE_TXN_TABLE_IF_TXNS_DISABLED(1078, "44A09", "Cannot create a transactional table if transactions are disabled"), - CANNOT_ALTER_TO_BE_TXN_IF_TXNS_DISABLED(1079, "44A10", "Cannot alter table to be transactional table if transactions are disabled"), - CANNOT_CREATE_TXN_TABLE_WITH_ROW_TIMESTAMP(1080, "44A11", "Cannot create a transactional table if transactions are disabled"), - CANNOT_ALTER_TO_BE_TXN_WITH_ROW_TIMESTAMP(1081, "44A12", "Cannot alter table to be transactional table if transactions are disabled"), - TX_MUST_BE_ENABLED_TO_SET_TX_CONTEXT(1082, "44A13", "Cannot set transaction context if transactions are disabled"), - TX_MUST_BE_ENABLED_TO_SET_AUTO_FLUSH(1083, "44A14", "Cannot set auto flush if transactions are disabled"), - TX_MUST_BE_ENABLED_TO_SET_ISOLATION_LEVEL(1084, "44A15", "Cannot set isolation level to TRANSACTION_READ_COMMITTED or TRANSACTION_SERIALIZABLE if transactions are disabled"), - TX_UNABLE_TO_GET_WRITE_FENCE(1085, "44A16", "Unable to obtain write fence for DDL operation"), + CANNOT_CREATE_TXN_TABLE_IF_TXNS_DISABLED(1078, "44A09", "Cannot create a transactional table if transactions are disabled."), + CANNOT_ALTER_TO_BE_TXN_IF_TXNS_DISABLED(1079, "44A10", "Cannot alter table to be transactional table if transactions are disabled."), + CANNOT_CREATE_TXN_TABLE_WITH_ROW_TIMESTAMP(1080, "44A11", "Cannot create a transactional table if transactions are disabled."), + CANNOT_ALTER_TO_BE_TXN_WITH_ROW_TIMESTAMP(1081, "44A12", "Cannot alter table to be transactional table if transactions are disabled."), + TX_MUST_BE_ENABLED_TO_SET_TX_CONTEXT(1082, "44A13", "Cannot set transaction context if transactions are disabled."), + TX_MUST_BE_ENABLED_TO_SET_AUTO_FLUSH(1083, "44A14", "Cannot set auto flush if transactions are disabled."), + TX_MUST_BE_ENABLED_TO_SET_ISOLATION_LEVEL(1084, "44A15", "Cannot set isolation level to TRANSACTION_READ_COMMITTED or TRANSACTION_SERIALIZABLE if transactions are disabled."), + TX_UNABLE_TO_GET_WRITE_FENCE(1085, "44A16", "Unable to obtain write fence for DDL operation."), /** Sequence related */ SEQUENCE_ALREADY_EXIST(1200, "42Z00", "Sequence already exists.", new Factory() { @@ -300,17 +300,17 @@ public SQLException newException(SQLExceptionInfo info) { CACHE_MUST_BE_NON_NEGATIVE_CONSTANT(1204, "42Z04", "Sequence CACHE value must be a non negative integer constant."), INVALID_USE_OF_NEXT_VALUE_FOR(1205, "42Z05", "NEXT VALUE FOR may only be used as in a SELECT or an UPSERT VALUES expression."), CANNOT_CALL_CURRENT_BEFORE_NEXT_VALUE(1206, "42Z06", "NEXT VALUE FOR must be called before CURRENT VALUE FOR is called."), - EMPTY_SEQUENCE_CACHE(1207, "42Z07", "No more cached sequence values"), + EMPTY_SEQUENCE_CACHE(1207, "42Z07", "No more cached sequence values."), MINVALUE_MUST_BE_CONSTANT(1208, "42Z08", "Sequence MINVALUE must be an integer or long constant."), MAXVALUE_MUST_BE_CONSTANT(1209, "42Z09", "Sequence MAXVALUE must be an integer or long constant."), MINVALUE_MUST_BE_LESS_THAN_OR_EQUAL_TO_MAXVALUE(1210, "42Z10", "Sequence MINVALUE must be less than or equal to MAXVALUE."), STARTS_WITH_MUST_BE_BETWEEN_MIN_MAX_VALUE(1211, "42Z11", - "STARTS WITH value must be greater than or equal to MINVALUE and less than or equal to MAXVALUE"), - SEQUENCE_VAL_REACHED_MAX_VALUE(1212, "42Z12", "Reached MAXVALUE of sequence"), - SEQUENCE_VAL_REACHED_MIN_VALUE(1213, "42Z13", "Reached MINVALUE of sequence"), - INCREMENT_BY_MUST_NOT_BE_ZERO(1214, "42Z14", "Sequence INCREMENT BY value cannot be zero"), + "STARTS WITH value must be greater than or equal to MINVALUE and less than or equal to MAXVALUE."), + SEQUENCE_VAL_REACHED_MAX_VALUE(1212, "42Z12", "Reached MAXVALUE of sequence."), + SEQUENCE_VAL_REACHED_MIN_VALUE(1213, "42Z13", "Reached MINVALUE of sequence."), + INCREMENT_BY_MUST_NOT_BE_ZERO(1214, "42Z14", "Sequence INCREMENT BY value cannot be zero."), NUM_SEQ_TO_ALLOCATE_MUST_BE_CONSTANT(1215, "42Z15", "Sequence NEXT n VALUES FOR must be a positive integer or constant." ), - NUM_SEQ_TO_ALLOCATE_NOT_SUPPORTED(1216, "42Z16", "Sequence NEXT n VALUES FOR is not supported for Sequences with the CYCLE flag" ), + NUM_SEQ_TO_ALLOCATE_NOT_SUPPORTED(1216, "42Z16", "Sequence NEXT n VALUES FOR is not supported for Sequences with the CYCLE flag." ), /** Parser error. (errorcode 06, sqlState 42P) */ PARSER_ERROR(601, "42P00", "Syntax error.", Factory.SYNTAX_ERROR), @@ -323,7 +323,7 @@ public SQLException newException(SQLExceptionInfo info) { * Implementation defined class. Execution exceptions (errorcode 11, sqlstate XCL). */ RESULTSET_CLOSED(1101, "XCL01", "ResultSet is closed."), - GET_TABLE_REGIONS_FAIL(1102, "XCL02", "Cannot get all table regions"), + GET_TABLE_REGIONS_FAIL(1102, "XCL02", "Cannot get all table regions."), EXECUTE_QUERY_NOT_APPLICABLE(1103, "XCL03", "executeQuery may not be used."), EXECUTE_UPDATE_NOT_APPLICABLE(1104, "XCL04", "executeUpdate may not be used."), SPLIT_POINT_NOT_CONSTANT(1105, "XCL05", "Split points must be constants."), @@ -335,22 +335,22 @@ public SQLException newException(SQLExceptionInfo info) { return new StaleRegionBoundaryCacheException(info.getSchemaName(), info.getTableName()); } }), - CANNOT_SPLIT_LOCAL_INDEX(1109,"XCL09", "Local index may not be pre-split"), - CANNOT_SALT_LOCAL_INDEX(1110,"XCL10", "Local index may not be salted"), + CANNOT_SPLIT_LOCAL_INDEX(1109,"XCL09", "Local index may not be pre-split."), + CANNOT_SALT_LOCAL_INDEX(1110,"XCL10", "Local index may not be salted."), /** * Implementation defined class. Phoenix internal error. (errorcode 20, sqlstate INT). */ CANNOT_CALL_METHOD_ON_TYPE(2001, "INT01", "Cannot call method on the argument type."), - CLASS_NOT_UNWRAPPABLE(2002, "INT03", "Class not unwrappable"), + CLASS_NOT_UNWRAPPABLE(2002, "INT03", "Class not unwrappable."), PARAM_INDEX_OUT_OF_BOUND(2003, "INT04", "Parameter position is out of range."), - PARAM_VALUE_UNBOUND(2004, "INT05", "Parameter value unbound"), + PARAM_VALUE_UNBOUND(2004, "INT05", "Parameter value unbound."), INTERRUPTED_EXCEPTION(2005, "INT07", "Interrupted exception."), INCOMPATIBLE_CLIENT_SERVER_JAR(2006, "INT08", "Incompatible jars detected between client and server."), OUTDATED_JARS(2007, "INT09", "Outdated jars."), INDEX_METADATA_NOT_FOUND(2008, "INT10", "Unable to find cached index metadata. "), - UNKNOWN_ERROR_CODE(2009, "INT11", "Unknown error code"), - OPERATION_TIMED_OUT(6000, "TIM01", "Operation timed out", new Factory() { + UNKNOWN_ERROR_CODE(2009, "INT11", "Unknown error code."), + OPERATION_TIMED_OUT(6000, "TIM01", "Operation timed out.", new Factory() { @Override public SQLException newException(SQLExceptionInfo info) { return new SQLTimeoutException(OPERATION_TIMED_OUT.getMessage(), diff --git a/phoenix-core/src/main/java/org/apache/phoenix/execute/MutationState.java b/phoenix-core/src/main/java/org/apache/phoenix/execute/MutationState.java index d3b36ec2551..3dfae466c52 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/execute/MutationState.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/execute/MutationState.java @@ -210,12 +210,12 @@ public long getMaxSize() { * when a data table transaction is started before the create index * but completes after it. In this case, we need to rerun the data * table transaction after the index creation so that the index rows - * are generated. See {@link #addReadFence(PTable)} and TEPHRA-157 + * are generated. See {@link #addDMLFence(PTable)} and TEPHRA-157 * for more information. * @param dataTable the data table upon which an index is being added * @throws SQLException */ - public void commitWriteFence(PTable dataTable) throws SQLException { + public void commitDDLFence(PTable dataTable) throws SQLException { if (dataTable.isTransactional()) { byte[] key = SchemaUtil.getTableKey(dataTable); boolean success = false; @@ -249,12 +249,12 @@ public void commitWriteFence(PTable dataTable) throws SQLException { /** * Add an entry to the change set representing the DML operation that is starting. * These entries will not conflict with each other, but they will conflict with a - * DDL operation of creating an index. See {@link #addReadFence(PTable)} and TEPHRA-157 + * DDL operation of creating an index. See {@link #addDMLFence(PTable)} and TEPHRA-157 * for more information. * @param dataTable the table which is doing DML * @throws SQLException */ - public void addReadFence(PTable dataTable) throws SQLException { + public void addDMLFence(PTable dataTable) throws SQLException { if (this.txContext == null) { throw new SQLExceptionInfo.Builder(SQLExceptionCode.NULL_TRANSACTION_CONTEXT).build().buildException(); } @@ -1004,7 +1004,9 @@ private void send(Iterator tableRefIterator) throws SQLException { // Keep all mutations we've encountered until a commit or rollback. // This is not ideal, but there's not good way to get the values back // in the event that we need to replay the commit. - joinMutationState(tableRef, valuesMap, txMutations); + // Copy TableRef so we have the original PTable and know when the + // indexes have changed. + joinMutationState(new TableRef(tableRef), valuesMap, txMutations); } // Remove batches as we process them if (sendAll) { @@ -1180,11 +1182,11 @@ public void commit() throws SQLException { Set txTableRefs = txMutations.keySet(); for (TableRef tableRef : txTableRefs) { PTable dataTable = tableRef.getTable(); - addReadFence(dataTable); + addDMLFence(dataTable); } try { // Only retry if an index was added - retryCommit = wasIndexAdded(txTableRefs); + retryCommit = shouldResubmitTransaction(txTableRefs); } catch (SQLException e) { retryCommit = false; if (sqlE == null) { @@ -1214,11 +1216,12 @@ public void commit() throws SQLException { * @return true if indexes were added and false otherwise. * @throws SQLException */ - private boolean wasIndexAdded(Set txTableRefs) throws SQLException { + private boolean shouldResubmitTransaction(Set txTableRefs) throws SQLException { if (logger.isInfoEnabled()) logger.info("Checking for index updates as of " + getInitialWritePointer()); MetaDataClient client = new MetaDataClient(connection); PMetaData cache = connection.getMetaDataCache(); - boolean addedIndexes = false; + boolean addedAnyIndexes = false; + boolean allImmutableTables = !txTableRefs.isEmpty(); for (TableRef tableRef : txTableRefs) { PTable dataTable = tableRef.getTable(); List oldIndexes; @@ -1227,20 +1230,24 @@ private boolean wasIndexAdded(Set txTableRefs) throws SQLException { MetaDataMutationResult result = client.updateCache(dataTable.getTenantId(), dataTable.getSchemaName().getString(), dataTable.getTableName().getString()); long timestamp = TransactionUtil.getResolvedTime(connection, result); tableRef.setTimeStamp(timestamp); - if (result.getTable() == null) { + PTable updatedDataTable = result.getTable(); + if (updatedDataTable == null) { throw new TableNotFoundException(dataTable.getSchemaName().getString(), dataTable.getTableName().getString()); } - tableRef.setTable(result.getTable()); - if (!addedIndexes) { + allImmutableTables |= updatedDataTable.isImmutableRows(); + tableRef.setTable(updatedDataTable); + if (!addedAnyIndexes) { // TODO: in theory we should do a deep equals check here, as it's possible // that an index was dropped and recreated with the same name but different // indexed/covered columns. - addedIndexes = (!oldIndexes.equals(result.getTable().getIndexes())); - if (logger.isInfoEnabled()) logger.info((addedIndexes ? "Updates " : "No updates ") + "as of " + timestamp + " to " + dataTable.getName().getString() + " with indexes " + tableRef.getTable().getIndexes()); + addedAnyIndexes = (!oldIndexes.equals(updatedDataTable.getIndexes())); + if (logger.isInfoEnabled()) logger.info((addedAnyIndexes ? "Updates " : "No updates ") + "as of " + timestamp + " to " + updatedDataTable.getName().getString() + " with indexes " + updatedDataTable.getIndexes()); } } - if (logger.isInfoEnabled()) logger.info((addedIndexes ? "Updates " : "No updates ") + "to indexes as of " + getInitialWritePointer()); - return addedIndexes; + if (logger.isInfoEnabled()) logger.info((addedAnyIndexes ? "Updates " : "No updates ") + "to indexes as of " + getInitialWritePointer() + " over " + (allImmutableTables ? " all immutable tables" : " some mutable tables")); + // If all tables are immutable, we know the conflict we got was due to our DDL/DML fence. + // If any indexes were added, then the conflict might be due to DDL/DML fence. + return allImmutableTables || addedAnyIndexes; } /** diff --git a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixStatement.java b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixStatement.java index 6bb57224b26..b54ccd5734d 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixStatement.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixStatement.java @@ -331,7 +331,7 @@ public Integer call() throws SQLException { MutationPlan plan = stmt.compilePlan(PhoenixStatement.this, Sequence.ValueOp.VALIDATE_SEQUENCE); if (plan.getTargetRef() != null && plan.getTargetRef().getTable() != null && plan.getTargetRef().getTable().isTransactional()) { state.startTransaction(); - state.addReadFence(plan.getTargetRef().getTable()); + state.addDMLFence(plan.getTargetRef().getTable()); } Iterator tableRefs = plan.getSourceRefs().iterator(); state.sendUncommitted(tableRefs); diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java index d134f0898a1..0b446b397a7 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java @@ -1074,7 +1074,7 @@ private MutationState buildIndex(PTable index, TableRef dataTableRef) throws SQL @Override public MutationState execute() throws SQLException { - connection.getMutationState().commitWriteFence(dataTable); + connection.getMutationState().commitDDLFence(dataTable); Cell kv = plan.iterator().next().getValue(0); ImmutableBytesWritable tmpPtr = new ImmutableBytesWritable(kv.getValueArray(), kv.getValueOffset(), kv.getValueLength()); // A single Cell will be returned with the count(*) - we decode that here diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/TableRef.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/TableRef.java index 35e2f779480..8f6e271d8ba 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/TableRef.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/TableRef.java @@ -36,12 +36,16 @@ public class TableRef { private final long lowerBoundTimeStamp; private final boolean hasDynamicCols; + public TableRef(TableRef tableRef) { + this(tableRef.alias, tableRef.table, tableRef.upperBoundTimeStamp, tableRef.lowerBoundTimeStamp, tableRef.hasDynamicCols); + } + public TableRef(TableRef tableRef, long timeStamp) { - this(tableRef.alias, tableRef.table, timeStamp, tableRef.hasDynamicCols); + this(tableRef.alias, tableRef.table, timeStamp, tableRef.lowerBoundTimeStamp, tableRef.hasDynamicCols); } public TableRef(TableRef tableRef, String alias) { - this(alias, tableRef.table, tableRef.upperBoundTimeStamp, tableRef.hasDynamicCols); + this(alias, tableRef.table, tableRef.upperBoundTimeStamp, tableRef.lowerBoundTimeStamp, tableRef.hasDynamicCols); } public TableRef(PTable table) { diff --git a/phoenix-core/src/test/java/org/apache/phoenix/compile/QueryCompilerTest.java b/phoenix-core/src/test/java/org/apache/phoenix/compile/QueryCompilerTest.java index 94115492707..6d6dcdff00e 100644 --- a/phoenix-core/src/test/java/org/apache/phoenix/compile/QueryCompilerTest.java +++ b/phoenix-core/src/test/java/org/apache/phoenix/compile/QueryCompilerTest.java @@ -217,7 +217,7 @@ public void testNoPK() throws Exception { statement.execute(); fail(); } catch (SQLException e) { - assertTrue(e.getMessage(), e.getMessage().contains("ERROR 517 (42895): Invalid not null constraint on non primary key column columnName=FOO.PK")); + assertEquals(SQLExceptionCode.INVALID_NOT_NULL_CONSTRAINT.getErrorCode(), e.getErrorCode()); } finally { conn.close(); } From 15432d1ac63ef429600333ad36c06f8e43deacf8 Mon Sep 17 00:00:00 2001 From: James Taylor Date: Tue, 19 Jan 2016 20:25:55 -0800 Subject: [PATCH 27/47] PHOENIX-2611 current_date/time functions produce invalid values --- .../it/java/org/apache/phoenix/end2end/DateTimeIT.java | 10 ++++++++++ .../main/java/org/apache/phoenix/schema/TableRef.java | 3 +-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/DateTimeIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/DateTimeIT.java index 0fce403af7d..6c53f1d541f 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/DateTimeIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/DateTimeIT.java @@ -52,6 +52,7 @@ import java.util.GregorianCalendar; import org.apache.phoenix.util.DateUtil; +import org.apache.phoenix.util.TestUtil; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -664,4 +665,13 @@ public void testNullDate() throws Exception { assertNull(rs.getDate(1, GregorianCalendar.getInstance())); assertFalse(rs.next()); } + + @Test + public void testCurrentDateWithNoTable() throws Exception { + long expectedTime = System.currentTimeMillis(); + ResultSet rs = conn.createStatement().executeQuery("SELECT CURRENT_DATE()"); + assertTrue(rs.next()); + long actualTime = rs.getDate(1).getTime(); + assertTrue(Math.abs(actualTime - expectedTime) < TestUtil.MILLIS_IN_DAY); + } } diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/TableRef.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/TableRef.java index 8f6e271d8ba..e2313421ae9 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/TableRef.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/TableRef.java @@ -19,7 +19,6 @@ import java.util.Objects; -import org.apache.hadoop.hbase.HConstants; import org.apache.phoenix.compile.TupleProjectionCompiler; import org.apache.phoenix.query.QueryConstants; import org.apache.phoenix.util.IndexUtil; @@ -49,7 +48,7 @@ public TableRef(TableRef tableRef, String alias) { } public TableRef(PTable table) { - this(null, table, HConstants.LATEST_TIMESTAMP, false); + this(null, table, QueryConstants.UNSET_TIMESTAMP, false); } public TableRef(PTable table, long upperBoundTimeStamp, long lowerBoundTimeStamp) { From 0b7a3c460e1171b5e67934684bf8ee71f07927cb Mon Sep 17 00:00:00 2001 From: Mujtaba Date: Mon, 5 Oct 2015 10:57:35 -0700 Subject: [PATCH 28/47] PHOENIX-2303 Fix for performance.py not using JAVA_HOME (bpanneton) --- bin/performance.py | 36 +++++++++++++++++++++++++++++++++--- 1 file changed, 33 insertions(+), 3 deletions(-) diff --git a/bin/performance.py b/bin/performance.py index 2f7fad1002f..fe39ddfc0f5 100755 --- a/bin/performance.py +++ b/bin/performance.py @@ -66,10 +66,40 @@ def createFileWithContent(filename, content): # HBase/Phoenix client side property override hbase_config_path = os.getenv('HBASE_CONF_DIR', phoenix_utils.current_dir) -execute = ('java -cp "%s%s%s" -Dlog4j.configuration=file:' + +java_home = os.getenv('JAVA_HOME') + +# load hbase-env.??? to extract JAVA_HOME, HBASE_PID_DIR, HBASE_LOG_DIR +hbase_env_path = None +hbase_env_cmd = None +if os.name == 'posix': + hbase_env_path = os.path.join(hbase_config_path, 'hbase-env.sh') + hbase_env_cmd = ['bash', '-c', 'source %s && env' % hbase_env_path] +elif os.name == 'nt': + hbase_env_path = os.path.join(hbase_config_path, 'hbase-env.cmd') + hbase_env_cmd = ['cmd.exe', '/c', 'call %s & set' % hbase_env_path] +if not hbase_env_path or not hbase_env_cmd: + print >> sys.stderr, "hbase-env file unknown on platform %s" % os.name + sys.exit(-1) + +hbase_env = {} +if os.path.isfile(hbase_env_path): + p = subprocess.Popen(hbase_env_cmd, stdout = subprocess.PIPE) + for x in p.stdout: + (k, _, v) = x.partition('=') + hbase_env[k.strip()] = v.strip() + +if hbase_env.has_key('JAVA_HOME'): + java_home = hbase_env['JAVA_HOME'] + +if java_home: + java_cmd = os.path.join(java_home, 'bin', 'java') +else: + java_cmd = 'java' + +execute = ('%s -cp "%s%s%s" -Dlog4j.configuration=file:' + os.path.join(phoenix_utils.current_dir, "log4j.properties") + ' org.apache.phoenix.util.PhoenixRuntime -t %s %s ') % \ - (hbase_config_path, os.pathsep, phoenix_utils.phoenix_client_jar, table, zookeeper) + (java_cmd, hbase_config_path, os.pathsep, phoenix_utils.phoenix_client_jar, table, zookeeper) # Create Table DDL createtable = "CREATE TABLE IF NOT EXISTS %s (HOST CHAR(2) NOT NULL,\ @@ -98,7 +128,7 @@ def createFileWithContent(filename, content): queryex("5 - Filter + Count", "SELECT COUNT(1) FROM %s WHERE CORE<10;" % (table)) print "\nGenerating and upserting data..." -exitcode = subprocess.call('java -jar %s %s %s' % (phoenix_utils.testjar, data, rowcount), +exitcode = subprocess.call('%s -jar %s %s %s' % (java_cmd, phoenix_utils.testjar, data, rowcount), shell=True) if exitcode != 0: sys.exit(exitcode) From 588f449b0bfc5ba6398c30ee76745a78c4996976 Mon Sep 17 00:00:00 2001 From: Nick Dimiduk Date: Tue, 19 Jan 2016 20:17:55 -0800 Subject: [PATCH 29/47] PHOENIX-2610 Allow passing PHOENIX_OPTS to command line utils --- bin/performance.py | 2 +- bin/phoenix_sandbox.py | 2 +- bin/psql.py | 3 ++- bin/queryserver.py | 2 +- bin/sqlline-thin.py | 3 ++- bin/sqlline.py | 3 ++- bin/traceserver.py | 3 ++- 7 files changed, 11 insertions(+), 7 deletions(-) diff --git a/bin/performance.py b/bin/performance.py index fe39ddfc0f5..c16dd5a31ae 100755 --- a/bin/performance.py +++ b/bin/performance.py @@ -96,7 +96,7 @@ def createFileWithContent(filename, content): else: java_cmd = 'java' -execute = ('%s -cp "%s%s%s" -Dlog4j.configuration=file:' + +execute = ('%s $PHOENIX_OPTS -cp "%s%s%s" -Dlog4j.configuration=file:' + os.path.join(phoenix_utils.current_dir, "log4j.properties") + ' org.apache.phoenix.util.PhoenixRuntime -t %s %s ') % \ (java_cmd, hbase_config_path, os.pathsep, phoenix_utils.phoenix_client_jar, table, zookeeper) diff --git a/bin/phoenix_sandbox.py b/bin/phoenix_sandbox.py index 433bc988b08..4279dd60b11 100755 --- a/bin/phoenix_sandbox.py +++ b/bin/phoenix_sandbox.py @@ -43,7 +43,7 @@ with open(cp_file_path, 'rb') as cp_file: cp_components.append(cp_file.read()) -java_cmd = ("java -Dlog4j.configuration=file:%s " + +java_cmd = ("java $PHOENIX_OPTS -Dlog4j.configuration=file:%s " + "-cp %s org.apache.phoenix.Sandbox") % ( logging_config, ":".join(cp_components)) diff --git a/bin/psql.py b/bin/psql.py index d4269d11ad0..973d3de4b06 100755 --- a/bin/psql.py +++ b/bin/psql.py @@ -62,7 +62,8 @@ else: java = 'java' -java_cmd = java +' -cp "' + phoenix_utils.hbase_conf_dir + os.pathsep + phoenix_utils.phoenix_client_jar + \ +java_cmd = java + ' $PHOENIX_OPTS ' + \ + ' -cp "' + phoenix_utils.hbase_conf_dir + os.pathsep + phoenix_utils.phoenix_client_jar + \ os.pathsep + phoenix_utils.hadoop_conf + os.pathsep + phoenix_utils.hadoop_classpath + '" -Dlog4j.configuration=file:' + \ os.path.join(phoenix_utils.current_dir, "log4j.properties") + \ " org.apache.phoenix.util.PhoenixRuntime " + args diff --git a/bin/queryserver.py b/bin/queryserver.py index 78d340c8607..d4228b39cf2 100755 --- a/bin/queryserver.py +++ b/bin/queryserver.py @@ -119,7 +119,7 @@ # " -Xdebug -Xrunjdwp:transport=dt_socket,address=5005,server=y,suspend=n " + \ # " -XX:+UnlockCommercialFeatures -XX:+FlightRecorder -XX:FlightRecorderOptions=defaultrecording=true,dumponexit=true" + \ -java_cmd = '%(java)s -cp ' + hbase_config_path + os.pathsep + hadoop_config_path + os.pathsep + \ +java_cmd = '%(java)s $PHOENIX_OPTS -cp ' + hbase_config_path + os.pathsep + hadoop_config_path + os.pathsep + \ phoenix_utils.phoenix_queryserver_jar + os.pathsep + phoenix_utils.phoenix_client_jar + \ " -Dproc_phoenixserver" + \ " -Dlog4j.configuration=file:" + os.path.join(phoenix_utils.current_dir, "log4j.properties") + \ diff --git a/bin/sqlline-thin.py b/bin/sqlline-thin.py index d17d756dd9d..b37cd90009d 100755 --- a/bin/sqlline-thin.py +++ b/bin/sqlline-thin.py @@ -145,7 +145,8 @@ def get_serialization(): else: java = 'java' -java_cmd = java + ' -cp "' + phoenix_utils.hbase_conf_dir + os.pathsep + phoenix_utils.phoenix_thin_client_jar + \ +java_cmd = java + ' $PHOENIX_OPTS ' + \ + ' -cp "' + phoenix_utils.hbase_conf_dir + os.pathsep + phoenix_utils.phoenix_thin_client_jar + \ os.pathsep + phoenix_utils.hadoop_conf + os.pathsep + phoenix_utils.hadoop_classpath + '" -Dlog4j.configuration=file:' + \ os.path.join(phoenix_utils.current_dir, "log4j.properties") + \ " sqlline.SqlLine -d org.apache.phoenix.queryserver.client.Driver " + \ diff --git a/bin/sqlline.py b/bin/sqlline.py index d1fdf741645..f14cfc7ed50 100755 --- a/bin/sqlline.py +++ b/bin/sqlline.py @@ -87,7 +87,8 @@ def kill_child(): if os.name == 'nt': colorSetting = "false" -java_cmd = java + ' -cp "' + phoenix_utils.hbase_conf_dir + os.pathsep + phoenix_utils.phoenix_client_jar + os.pathsep + phoenix_utils.hadoop_common_jar + os.pathsep + phoenix_utils.hadoop_hdfs_jar + \ +java_cmd = java + ' $PHOENIX_OPTS ' + \ + ' -cp "' + phoenix_utils.hbase_conf_dir + os.pathsep + phoenix_utils.phoenix_client_jar + os.pathsep + phoenix_utils.hadoop_common_jar + os.pathsep + phoenix_utils.hadoop_hdfs_jar + \ os.pathsep + phoenix_utils.hadoop_conf + os.pathsep + phoenix_utils.hadoop_classpath + '" -Dlog4j.configuration=file:' + \ os.path.join(phoenix_utils.current_dir, "log4j.properties") + \ " sqlline.SqlLine -d org.apache.phoenix.jdbc.PhoenixDriver \ diff --git a/bin/traceserver.py b/bin/traceserver.py index 082b4173358..665099eed2f 100755 --- a/bin/traceserver.py +++ b/bin/traceserver.py @@ -116,7 +116,8 @@ # " -Xdebug -Xrunjdwp:transport=dt_socket,address=5005,server=y,suspend=n " + \ # " -XX:+UnlockCommercialFeatures -XX:+FlightRecorder -XX:FlightRecorderOptions=defaultrecording=true,dumponexit=true" + \ -java_cmd = '%(java)s -cp ' + hbase_config_path + os.pathsep + phoenix_utils.phoenix_traceserver_jar + os.pathsep + phoenix_utils.phoenix_client_jar + \ +java_cmd = '%(java)s $PHOENIX_OPTS ' + \ + '-cp ' + hbase_config_path + os.pathsep + phoenix_utils.phoenix_traceserver_jar + os.pathsep + phoenix_utils.phoenix_client_jar + \ " -Dproc_phoenixtraceserver" + \ " -Dlog4j.configuration=file:" + os.path.join(phoenix_utils.current_dir, "log4j.properties") + \ " -Dpsql.root.logger=%(root_logger)s" + \ From b63ca5fbdf60e6dd9a2d0d054c3be058e3b6b5c5 Mon Sep 17 00:00:00 2001 From: Nick Dimiduk Date: Tue, 19 Jan 2016 20:20:36 -0800 Subject: [PATCH 30/47] PHOENIX-2608 Incompatibility between Jackson1 version shipped with Phoenix, Yarn (Ted Yu) --- pom.xml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 72551a46543..2c47ca27dae 100644 --- a/pom.xml +++ b/pom.xml @@ -87,7 +87,7 @@ 1.2 2.5.1 0.13.0 - 1.8.8 + 1.9.2 3.5 1.2.17 1.6.4 @@ -99,7 +99,6 @@ 1.0 1.1.8 13.0.1 - 1.8.8 1.4.0 1.3.2 2.11 From 4d323b66b8f685872c7b94f3d243c940f7fa37d9 Mon Sep 17 00:00:00 2001 From: James Taylor Date: Tue, 19 Jan 2016 22:10:48 -0800 Subject: [PATCH 31/47] PHOENIX-2478 Rows committed in transaction overlapping index creation are not populated --- .../src/main/java/org/apache/phoenix/execute/MutationState.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/phoenix-core/src/main/java/org/apache/phoenix/execute/MutationState.java b/phoenix-core/src/main/java/org/apache/phoenix/execute/MutationState.java index 3dfae466c52..f1a9c0234f7 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/execute/MutationState.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/execute/MutationState.java @@ -1234,7 +1234,7 @@ private boolean shouldResubmitTransaction(Set txTableRefs) throws SQLE if (updatedDataTable == null) { throw new TableNotFoundException(dataTable.getSchemaName().getString(), dataTable.getTableName().getString()); } - allImmutableTables |= updatedDataTable.isImmutableRows(); + allImmutableTables &= updatedDataTable.isImmutableRows(); tableRef.setTable(updatedDataTable); if (!addedAnyIndexes) { // TODO: in theory we should do a deep equals check here, as it's possible From 3057ee06aa072492c69e519265c8c643ff9e11fd Mon Sep 17 00:00:00 2001 From: Ankit Singhal Date: Wed, 20 Jan 2016 15:15:14 +0530 Subject: [PATCH 32/47] updatd review comments for protobuf backward compatibility and moving code to upgrade Util --- .../coprocessor/MetaDataRegionObserver.java | 86 +--- .../generated/PGuidePostsProtos.java | 371 +++++++++++++----- .../coprocessor/generated/PTableProtos.java | 130 +++--- .../org/apache/phoenix/schema/PTableImpl.java | 10 +- .../org/apache/phoenix/util/UpgradeUtil.java | 66 ++++ phoenix-protocol/src/main/PGuidePosts.proto | 5 +- phoenix-protocol/src/main/PTable.proto | 2 +- 7 files changed, 412 insertions(+), 258 deletions(-) diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataRegionObserver.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataRegionObserver.java index 64d5f3cef34..7950ac8f183 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataRegionObserver.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataRegionObserver.java @@ -32,15 +32,9 @@ import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CoprocessorEnvironment; import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.client.Delete; -import org.apache.hadoop.hbase.client.Get; import org.apache.hadoop.hbase.client.HTableInterface; -import org.apache.hadoop.hbase.client.Mutation; -import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.Result; -import org.apache.hadoop.hbase.client.ResultScanner; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver; import org.apache.hadoop.hbase.coprocessor.ObserverContext; @@ -55,7 +49,6 @@ import org.apache.phoenix.jdbc.PhoenixConnection; import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData; import org.apache.phoenix.jdbc.PhoenixDriver; -import org.apache.phoenix.query.QueryConstants; import org.apache.phoenix.query.QueryServices; import org.apache.phoenix.query.QueryServicesOptions; import org.apache.phoenix.schema.MetaDataClient; @@ -63,14 +56,10 @@ import org.apache.phoenix.schema.PTable; import org.apache.phoenix.schema.TableRef; import org.apache.phoenix.schema.types.PLong; -import org.apache.phoenix.schema.types.PVarchar; -import org.apache.phoenix.util.ByteUtil; -import org.apache.phoenix.util.KeyValueUtil; import org.apache.phoenix.util.MetaDataUtil; import org.apache.phoenix.util.PhoenixRuntime; import org.apache.phoenix.util.SchemaUtil; - -import com.google.common.collect.Lists; +import org.apache.phoenix.util.UpgradeUtil; /** @@ -82,8 +71,6 @@ public class MetaDataRegionObserver extends BaseRegionObserver { protected ScheduledThreadPoolExecutor executor = new ScheduledThreadPoolExecutor(1); private boolean enableRebuildIndex = QueryServicesOptions.DEFAULT_INDEX_FAILURE_HANDLING_REBUILD; private long rebuildIndexTimeInterval = QueryServicesOptions.DEFAULT_INDEX_FAILURE_HANDLING_REBUILD_INTERVAL; - public static final byte[] UPGRADE_TO_4_7_COLUMN_NAME = Bytes.toBytes("UPGRADE_TO_4_7"); - public static final String UPGRADE_TO_4_7_COLUMN_VALUE = "TRUNCATED_4_7"; @Override public void preClose(final ObserverContext c, @@ -134,72 +121,15 @@ public void run() { HTableInterface statsTable = null; try { Thread.sleep(1000); - LOG.info("Stats will be deleted for upgrade 4.7 requirement!!"); metaTable = env.getTable(TableName.valueOf(PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME)); - List columnCells = metaTable - .get(new Get(SchemaUtil.getTableKey(null, PhoenixDatabaseMetaData.SYSTEM_SCHEMA_NAME, - PhoenixDatabaseMetaData.SYSTEM_CATALOG_TABLE))) - .getColumnCells(PhoenixDatabaseMetaData.TABLE_FAMILY_BYTES, - QueryConstants.EMPTY_COLUMN_BYTES); - if (!columnCells.isEmpty() - && columnCells.get(0).getTimestamp() < MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_7_0) { - - statsTable = env.getTable(TableName.valueOf(PhoenixDatabaseMetaData.SYSTEM_STATS_NAME)); - byte[] statsTableKey = SchemaUtil.getTableKey(null, PhoenixDatabaseMetaData.SYSTEM_SCHEMA_NAME, - PhoenixDatabaseMetaData.SYSTEM_STATS_TABLE); - KeyValue upgradeKV = KeyValueUtil.newKeyValue(statsTableKey, - PhoenixDatabaseMetaData.TABLE_FAMILY_BYTES, UPGRADE_TO_4_7_COLUMN_NAME, - MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_7_0 - 1, - PVarchar.INSTANCE.toBytes(UPGRADE_TO_4_7_COLUMN_VALUE)); - Put upgradePut = new Put(statsTableKey); - upgradePut.add(upgradeKV); - - // check for null in UPGRADE_TO_4_7_COLUMN_NAME in checkAndPut so that only single client - // drop the rows of SYSTEM.STATS - if (metaTable.checkAndPut(statsTableKey, PhoenixDatabaseMetaData.TABLE_FAMILY_BYTES, - UPGRADE_TO_4_7_COLUMN_NAME, null, upgradePut)) { - List mutations = Lists.newArrayListWithExpectedSize(1000); - Scan scan = new Scan(); - scan.setRaw(true); - scan.setMaxVersions(); - ResultScanner statsScanner = statsTable.getScanner(scan); - Result r; - mutations.clear(); - int count = 0; - while ((r = statsScanner.next()) != null) { - Delete delete = null; - for (KeyValue keyValue : r.raw()) { - if (KeyValue.Type.codeToType(keyValue.getType()) == KeyValue.Type.Put) { - if (delete == null) { - delete = new Delete(keyValue.getRow()); - } - KeyValue deleteKeyValue = new KeyValue(keyValue.getRowArray(), - keyValue.getRowOffset(), keyValue.getRowLength(), - keyValue.getFamilyArray(), keyValue.getFamilyOffset(), - keyValue.getFamilyLength(), keyValue.getQualifierArray(), - keyValue.getQualifierOffset(), keyValue.getQualifierLength(), - keyValue.getTimestamp(), KeyValue.Type.Delete, - ByteUtil.EMPTY_BYTE_ARRAY, 0, 0); - delete.addDeleteMarker(deleteKeyValue); - } - } - if (delete != null) { - mutations.add(delete); - if (count > 10) { - statsTable.batch(mutations); - mutations.clear(); - count = 0; - } - count++; - } - } - if (!mutations.isEmpty()) { - statsTable.batch(mutations); - } - } + statsTable = env.getTable(TableName.valueOf(PhoenixDatabaseMetaData.SYSTEM_STATS_NAME)); + if (UpgradeUtil.truncateStats(metaTable, statsTable)) { + LOG.info("Stats are successfully truncated for upgrade 4.7!!"); } - } catch (Exception e) { - LOG.warn("Exception while deleting stats..", e); + } catch (Exception exception) { + LOG.warn("Exception while truncate stats..," + + " please check and delete stats manually inorder to get proper result with old client!!"); + LOG.warn(exception.getStackTrace()); } finally { try { if (metaTable != null) { diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/PGuidePostsProtos.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/PGuidePostsProtos.java index 58999740ed8..d9dcb56c4ab 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/PGuidePostsProtos.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/PGuidePostsProtos.java @@ -11,15 +11,19 @@ public static void registerAllExtensions( public interface PGuidePostsOrBuilder extends com.google.protobuf.MessageOrBuilder { - // optional bytes guidePosts = 1; + // repeated bytes guidePosts = 1; /** - * optional bytes guidePosts = 1; + * repeated bytes guidePosts = 1; */ - boolean hasGuidePosts(); + java.util.List getGuidePostsList(); /** - * optional bytes guidePosts = 1; + * repeated bytes guidePosts = 1; */ - com.google.protobuf.ByteString getGuidePosts(); + int getGuidePostsCount(); + /** + * repeated bytes guidePosts = 1; + */ + com.google.protobuf.ByteString getGuidePosts(int index); // optional int64 byteCount = 2; /** @@ -51,15 +55,25 @@ public interface PGuidePostsOrBuilder */ int getMaxLength(); - // optional int32 guidePostsCount = 5; + // optional int32 encodedGuidePostsCount = 5; /** - * optional int32 guidePostsCount = 5; + * optional int32 encodedGuidePostsCount = 5; */ - boolean hasGuidePostsCount(); + boolean hasEncodedGuidePostsCount(); /** - * optional int32 guidePostsCount = 5; + * optional int32 encodedGuidePostsCount = 5; */ - int getGuidePostsCount(); + int getEncodedGuidePostsCount(); + + // optional bytes encodedGuidePosts = 6; + /** + * optional bytes encodedGuidePosts = 6; + */ + boolean hasEncodedGuidePosts(); + /** + * optional bytes encodedGuidePosts = 6; + */ + com.google.protobuf.ByteString getEncodedGuidePosts(); } /** * Protobuf type {@code PGuidePosts} @@ -113,28 +127,36 @@ private PGuidePosts( break; } case 10: { - bitField0_ |= 0x00000001; - guidePosts_ = input.readBytes(); + if (!((mutable_bitField0_ & 0x00000001) == 0x00000001)) { + guidePosts_ = new java.util.ArrayList(); + mutable_bitField0_ |= 0x00000001; + } + guidePosts_.add(input.readBytes()); break; } case 16: { - bitField0_ |= 0x00000002; + bitField0_ |= 0x00000001; byteCount_ = input.readInt64(); break; } case 24: { - bitField0_ |= 0x00000004; + bitField0_ |= 0x00000002; rowCount_ = input.readInt64(); break; } case 32: { - bitField0_ |= 0x00000008; + bitField0_ |= 0x00000004; maxLength_ = input.readInt32(); break; } case 40: { + bitField0_ |= 0x00000008; + encodedGuidePostsCount_ = input.readInt32(); + break; + } + case 50: { bitField0_ |= 0x00000010; - guidePostsCount_ = input.readInt32(); + encodedGuidePosts_ = input.readBytes(); break; } } @@ -145,6 +167,9 @@ private PGuidePosts( throw new com.google.protobuf.InvalidProtocolBufferException( e.getMessage()).setUnfinishedMessage(this); } finally { + if (((mutable_bitField0_ & 0x00000001) == 0x00000001)) { + guidePosts_ = java.util.Collections.unmodifiableList(guidePosts_); + } this.unknownFields = unknownFields.build(); makeExtensionsImmutable(); } @@ -177,20 +202,27 @@ public com.google.protobuf.Parser getParserForType() { } private int bitField0_; - // optional bytes guidePosts = 1; + // repeated bytes guidePosts = 1; public static final int GUIDEPOSTS_FIELD_NUMBER = 1; - private com.google.protobuf.ByteString guidePosts_; + private java.util.List guidePosts_; /** - * optional bytes guidePosts = 1; + * repeated bytes guidePosts = 1; */ - public boolean hasGuidePosts() { - return ((bitField0_ & 0x00000001) == 0x00000001); + public java.util.List + getGuidePostsList() { + return guidePosts_; } /** - * optional bytes guidePosts = 1; + * repeated bytes guidePosts = 1; */ - public com.google.protobuf.ByteString getGuidePosts() { - return guidePosts_; + public int getGuidePostsCount() { + return guidePosts_.size(); + } + /** + * repeated bytes guidePosts = 1; + */ + public com.google.protobuf.ByteString getGuidePosts(int index) { + return guidePosts_.get(index); } // optional int64 byteCount = 2; @@ -200,7 +232,7 @@ public com.google.protobuf.ByteString getGuidePosts() { * optional int64 byteCount = 2; */ public boolean hasByteCount() { - return ((bitField0_ & 0x00000002) == 0x00000002); + return ((bitField0_ & 0x00000001) == 0x00000001); } /** * optional int64 byteCount = 2; @@ -216,7 +248,7 @@ public long getByteCount() { * optional int64 rowCount = 3; */ public boolean hasRowCount() { - return ((bitField0_ & 0x00000004) == 0x00000004); + return ((bitField0_ & 0x00000002) == 0x00000002); } /** * optional int64 rowCount = 3; @@ -232,7 +264,7 @@ public long getRowCount() { * optional int32 maxLength = 4; */ public boolean hasMaxLength() { - return ((bitField0_ & 0x00000008) == 0x00000008); + return ((bitField0_ & 0x00000004) == 0x00000004); } /** * optional int32 maxLength = 4; @@ -241,28 +273,45 @@ public int getMaxLength() { return maxLength_; } - // optional int32 guidePostsCount = 5; - public static final int GUIDEPOSTSCOUNT_FIELD_NUMBER = 5; - private int guidePostsCount_; + // optional int32 encodedGuidePostsCount = 5; + public static final int ENCODEDGUIDEPOSTSCOUNT_FIELD_NUMBER = 5; + private int encodedGuidePostsCount_; + /** + * optional int32 encodedGuidePostsCount = 5; + */ + public boolean hasEncodedGuidePostsCount() { + return ((bitField0_ & 0x00000008) == 0x00000008); + } + /** + * optional int32 encodedGuidePostsCount = 5; + */ + public int getEncodedGuidePostsCount() { + return encodedGuidePostsCount_; + } + + // optional bytes encodedGuidePosts = 6; + public static final int ENCODEDGUIDEPOSTS_FIELD_NUMBER = 6; + private com.google.protobuf.ByteString encodedGuidePosts_; /** - * optional int32 guidePostsCount = 5; + * optional bytes encodedGuidePosts = 6; */ - public boolean hasGuidePostsCount() { + public boolean hasEncodedGuidePosts() { return ((bitField0_ & 0x00000010) == 0x00000010); } /** - * optional int32 guidePostsCount = 5; + * optional bytes encodedGuidePosts = 6; */ - public int getGuidePostsCount() { - return guidePostsCount_; + public com.google.protobuf.ByteString getEncodedGuidePosts() { + return encodedGuidePosts_; } private void initFields() { - guidePosts_ = com.google.protobuf.ByteString.EMPTY; + guidePosts_ = java.util.Collections.emptyList(); byteCount_ = 0L; rowCount_ = 0L; maxLength_ = 0; - guidePostsCount_ = 0; + encodedGuidePostsCount_ = 0; + encodedGuidePosts_ = com.google.protobuf.ByteString.EMPTY; } private byte memoizedIsInitialized = -1; public final boolean isInitialized() { @@ -276,20 +325,23 @@ public final boolean isInitialized() { public void writeTo(com.google.protobuf.CodedOutputStream output) throws java.io.IOException { getSerializedSize(); + for (int i = 0; i < guidePosts_.size(); i++) { + output.writeBytes(1, guidePosts_.get(i)); + } if (((bitField0_ & 0x00000001) == 0x00000001)) { - output.writeBytes(1, guidePosts_); + output.writeInt64(2, byteCount_); } if (((bitField0_ & 0x00000002) == 0x00000002)) { - output.writeInt64(2, byteCount_); + output.writeInt64(3, rowCount_); } if (((bitField0_ & 0x00000004) == 0x00000004)) { - output.writeInt64(3, rowCount_); + output.writeInt32(4, maxLength_); } if (((bitField0_ & 0x00000008) == 0x00000008)) { - output.writeInt32(4, maxLength_); + output.writeInt32(5, encodedGuidePostsCount_); } if (((bitField0_ & 0x00000010) == 0x00000010)) { - output.writeInt32(5, guidePostsCount_); + output.writeBytes(6, encodedGuidePosts_); } getUnknownFields().writeTo(output); } @@ -300,25 +352,34 @@ public int getSerializedSize() { if (size != -1) return size; size = 0; + { + int dataSize = 0; + for (int i = 0; i < guidePosts_.size(); i++) { + dataSize += com.google.protobuf.CodedOutputStream + .computeBytesSizeNoTag(guidePosts_.get(i)); + } + size += dataSize; + size += 1 * getGuidePostsList().size(); + } if (((bitField0_ & 0x00000001) == 0x00000001)) { size += com.google.protobuf.CodedOutputStream - .computeBytesSize(1, guidePosts_); + .computeInt64Size(2, byteCount_); } if (((bitField0_ & 0x00000002) == 0x00000002)) { size += com.google.protobuf.CodedOutputStream - .computeInt64Size(2, byteCount_); + .computeInt64Size(3, rowCount_); } if (((bitField0_ & 0x00000004) == 0x00000004)) { size += com.google.protobuf.CodedOutputStream - .computeInt64Size(3, rowCount_); + .computeInt32Size(4, maxLength_); } if (((bitField0_ & 0x00000008) == 0x00000008)) { size += com.google.protobuf.CodedOutputStream - .computeInt32Size(4, maxLength_); + .computeInt32Size(5, encodedGuidePostsCount_); } if (((bitField0_ & 0x00000010) == 0x00000010)) { size += com.google.protobuf.CodedOutputStream - .computeInt32Size(5, guidePostsCount_); + .computeBytesSize(6, encodedGuidePosts_); } size += getUnknownFields().getSerializedSize(); memoizedSerializedSize = size; @@ -343,11 +404,8 @@ public boolean equals(final java.lang.Object obj) { org.apache.phoenix.coprocessor.generated.PGuidePostsProtos.PGuidePosts other = (org.apache.phoenix.coprocessor.generated.PGuidePostsProtos.PGuidePosts) obj; boolean result = true; - result = result && (hasGuidePosts() == other.hasGuidePosts()); - if (hasGuidePosts()) { - result = result && getGuidePosts() - .equals(other.getGuidePosts()); - } + result = result && getGuidePostsList() + .equals(other.getGuidePostsList()); result = result && (hasByteCount() == other.hasByteCount()); if (hasByteCount()) { result = result && (getByteCount() @@ -363,10 +421,15 @@ public boolean equals(final java.lang.Object obj) { result = result && (getMaxLength() == other.getMaxLength()); } - result = result && (hasGuidePostsCount() == other.hasGuidePostsCount()); - if (hasGuidePostsCount()) { - result = result && (getGuidePostsCount() - == other.getGuidePostsCount()); + result = result && (hasEncodedGuidePostsCount() == other.hasEncodedGuidePostsCount()); + if (hasEncodedGuidePostsCount()) { + result = result && (getEncodedGuidePostsCount() + == other.getEncodedGuidePostsCount()); + } + result = result && (hasEncodedGuidePosts() == other.hasEncodedGuidePosts()); + if (hasEncodedGuidePosts()) { + result = result && getEncodedGuidePosts() + .equals(other.getEncodedGuidePosts()); } result = result && getUnknownFields().equals(other.getUnknownFields()); @@ -381,9 +444,9 @@ public int hashCode() { } int hash = 41; hash = (19 * hash) + getDescriptorForType().hashCode(); - if (hasGuidePosts()) { + if (getGuidePostsCount() > 0) { hash = (37 * hash) + GUIDEPOSTS_FIELD_NUMBER; - hash = (53 * hash) + getGuidePosts().hashCode(); + hash = (53 * hash) + getGuidePostsList().hashCode(); } if (hasByteCount()) { hash = (37 * hash) + BYTECOUNT_FIELD_NUMBER; @@ -397,9 +460,13 @@ public int hashCode() { hash = (37 * hash) + MAXLENGTH_FIELD_NUMBER; hash = (53 * hash) + getMaxLength(); } - if (hasGuidePostsCount()) { - hash = (37 * hash) + GUIDEPOSTSCOUNT_FIELD_NUMBER; - hash = (53 * hash) + getGuidePostsCount(); + if (hasEncodedGuidePostsCount()) { + hash = (37 * hash) + ENCODEDGUIDEPOSTSCOUNT_FIELD_NUMBER; + hash = (53 * hash) + getEncodedGuidePostsCount(); + } + if (hasEncodedGuidePosts()) { + hash = (37 * hash) + ENCODEDGUIDEPOSTS_FIELD_NUMBER; + hash = (53 * hash) + getEncodedGuidePosts().hashCode(); } hash = (29 * hash) + getUnknownFields().hashCode(); memoizedHashCode = hash; @@ -510,7 +577,7 @@ private static Builder create() { public Builder clear() { super.clear(); - guidePosts_ = com.google.protobuf.ByteString.EMPTY; + guidePosts_ = java.util.Collections.emptyList(); bitField0_ = (bitField0_ & ~0x00000001); byteCount_ = 0L; bitField0_ = (bitField0_ & ~0x00000002); @@ -518,8 +585,10 @@ public Builder clear() { bitField0_ = (bitField0_ & ~0x00000004); maxLength_ = 0; bitField0_ = (bitField0_ & ~0x00000008); - guidePostsCount_ = 0; + encodedGuidePostsCount_ = 0; bitField0_ = (bitField0_ & ~0x00000010); + encodedGuidePosts_ = com.google.protobuf.ByteString.EMPTY; + bitField0_ = (bitField0_ & ~0x00000020); return this; } @@ -548,26 +617,31 @@ public org.apache.phoenix.coprocessor.generated.PGuidePostsProtos.PGuidePosts bu org.apache.phoenix.coprocessor.generated.PGuidePostsProtos.PGuidePosts result = new org.apache.phoenix.coprocessor.generated.PGuidePostsProtos.PGuidePosts(this); int from_bitField0_ = bitField0_; int to_bitField0_ = 0; - if (((from_bitField0_ & 0x00000001) == 0x00000001)) { - to_bitField0_ |= 0x00000001; + if (((bitField0_ & 0x00000001) == 0x00000001)) { + guidePosts_ = java.util.Collections.unmodifiableList(guidePosts_); + bitField0_ = (bitField0_ & ~0x00000001); } result.guidePosts_ = guidePosts_; if (((from_bitField0_ & 0x00000002) == 0x00000002)) { - to_bitField0_ |= 0x00000002; + to_bitField0_ |= 0x00000001; } result.byteCount_ = byteCount_; if (((from_bitField0_ & 0x00000004) == 0x00000004)) { - to_bitField0_ |= 0x00000004; + to_bitField0_ |= 0x00000002; } result.rowCount_ = rowCount_; if (((from_bitField0_ & 0x00000008) == 0x00000008)) { - to_bitField0_ |= 0x00000008; + to_bitField0_ |= 0x00000004; } result.maxLength_ = maxLength_; if (((from_bitField0_ & 0x00000010) == 0x00000010)) { + to_bitField0_ |= 0x00000008; + } + result.encodedGuidePostsCount_ = encodedGuidePostsCount_; + if (((from_bitField0_ & 0x00000020) == 0x00000020)) { to_bitField0_ |= 0x00000010; } - result.guidePostsCount_ = guidePostsCount_; + result.encodedGuidePosts_ = encodedGuidePosts_; result.bitField0_ = to_bitField0_; onBuilt(); return result; @@ -584,8 +658,15 @@ public Builder mergeFrom(com.google.protobuf.Message other) { public Builder mergeFrom(org.apache.phoenix.coprocessor.generated.PGuidePostsProtos.PGuidePosts other) { if (other == org.apache.phoenix.coprocessor.generated.PGuidePostsProtos.PGuidePosts.getDefaultInstance()) return this; - if (other.hasGuidePosts()) { - setGuidePosts(other.getGuidePosts()); + if (!other.guidePosts_.isEmpty()) { + if (guidePosts_.isEmpty()) { + guidePosts_ = other.guidePosts_; + bitField0_ = (bitField0_ & ~0x00000001); + } else { + ensureGuidePostsIsMutable(); + guidePosts_.addAll(other.guidePosts_); + } + onChanged(); } if (other.hasByteCount()) { setByteCount(other.getByteCount()); @@ -596,8 +677,11 @@ public Builder mergeFrom(org.apache.phoenix.coprocessor.generated.PGuidePostsPro if (other.hasMaxLength()) { setMaxLength(other.getMaxLength()); } - if (other.hasGuidePostsCount()) { - setGuidePostsCount(other.getGuidePostsCount()); + if (other.hasEncodedGuidePostsCount()) { + setEncodedGuidePostsCount(other.getEncodedGuidePostsCount()); + } + if (other.hasEncodedGuidePosts()) { + setEncodedGuidePosts(other.getEncodedGuidePosts()); } this.mergeUnknownFields(other.getUnknownFields()); return this; @@ -626,38 +710,74 @@ public Builder mergeFrom( } private int bitField0_; - // optional bytes guidePosts = 1; - private com.google.protobuf.ByteString guidePosts_ = com.google.protobuf.ByteString.EMPTY; + // repeated bytes guidePosts = 1; + private java.util.List guidePosts_ = java.util.Collections.emptyList(); + private void ensureGuidePostsIsMutable() { + if (!((bitField0_ & 0x00000001) == 0x00000001)) { + guidePosts_ = new java.util.ArrayList(guidePosts_); + bitField0_ |= 0x00000001; + } + } /** - * optional bytes guidePosts = 1; + * repeated bytes guidePosts = 1; */ - public boolean hasGuidePosts() { - return ((bitField0_ & 0x00000001) == 0x00000001); + public java.util.List + getGuidePostsList() { + return java.util.Collections.unmodifiableList(guidePosts_); + } + /** + * repeated bytes guidePosts = 1; + */ + public int getGuidePostsCount() { + return guidePosts_.size(); } /** - * optional bytes guidePosts = 1; + * repeated bytes guidePosts = 1; */ - public com.google.protobuf.ByteString getGuidePosts() { - return guidePosts_; + public com.google.protobuf.ByteString getGuidePosts(int index) { + return guidePosts_.get(index); } /** - * optional bytes guidePosts = 1; + * repeated bytes guidePosts = 1; */ - public Builder setGuidePosts(com.google.protobuf.ByteString value) { + public Builder setGuidePosts( + int index, com.google.protobuf.ByteString value) { if (value == null) { throw new NullPointerException(); } - bitField0_ |= 0x00000001; - guidePosts_ = value; + ensureGuidePostsIsMutable(); + guidePosts_.set(index, value); onChanged(); return this; } /** - * optional bytes guidePosts = 1; + * repeated bytes guidePosts = 1; + */ + public Builder addGuidePosts(com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + ensureGuidePostsIsMutable(); + guidePosts_.add(value); + onChanged(); + return this; + } + /** + * repeated bytes guidePosts = 1; + */ + public Builder addAllGuidePosts( + java.lang.Iterable values) { + ensureGuidePostsIsMutable(); + super.addAll(values, guidePosts_); + onChanged(); + return this; + } + /** + * repeated bytes guidePosts = 1; */ public Builder clearGuidePosts() { + guidePosts_ = java.util.Collections.emptyList(); bitField0_ = (bitField0_ & ~0x00000001); - guidePosts_ = getDefaultInstance().getGuidePosts(); onChanged(); return this; } @@ -761,35 +881,71 @@ public Builder clearMaxLength() { return this; } - // optional int32 guidePostsCount = 5; - private int guidePostsCount_ ; + // optional int32 encodedGuidePostsCount = 5; + private int encodedGuidePostsCount_ ; /** - * optional int32 guidePostsCount = 5; + * optional int32 encodedGuidePostsCount = 5; */ - public boolean hasGuidePostsCount() { + public boolean hasEncodedGuidePostsCount() { return ((bitField0_ & 0x00000010) == 0x00000010); } /** - * optional int32 guidePostsCount = 5; + * optional int32 encodedGuidePostsCount = 5; */ - public int getGuidePostsCount() { - return guidePostsCount_; + public int getEncodedGuidePostsCount() { + return encodedGuidePostsCount_; } /** - * optional int32 guidePostsCount = 5; + * optional int32 encodedGuidePostsCount = 5; */ - public Builder setGuidePostsCount(int value) { + public Builder setEncodedGuidePostsCount(int value) { bitField0_ |= 0x00000010; - guidePostsCount_ = value; + encodedGuidePostsCount_ = value; onChanged(); return this; } /** - * optional int32 guidePostsCount = 5; + * optional int32 encodedGuidePostsCount = 5; */ - public Builder clearGuidePostsCount() { + public Builder clearEncodedGuidePostsCount() { bitField0_ = (bitField0_ & ~0x00000010); - guidePostsCount_ = 0; + encodedGuidePostsCount_ = 0; + onChanged(); + return this; + } + + // optional bytes encodedGuidePosts = 6; + private com.google.protobuf.ByteString encodedGuidePosts_ = com.google.protobuf.ByteString.EMPTY; + /** + * optional bytes encodedGuidePosts = 6; + */ + public boolean hasEncodedGuidePosts() { + return ((bitField0_ & 0x00000020) == 0x00000020); + } + /** + * optional bytes encodedGuidePosts = 6; + */ + public com.google.protobuf.ByteString getEncodedGuidePosts() { + return encodedGuidePosts_; + } + /** + * optional bytes encodedGuidePosts = 6; + */ + public Builder setEncodedGuidePosts(com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000020; + encodedGuidePosts_ = value; + onChanged(); + return this; + } + /** + * optional bytes encodedGuidePosts = 6; + */ + public Builder clearEncodedGuidePosts() { + bitField0_ = (bitField0_ & ~0x00000020); + encodedGuidePosts_ = getDefaultInstance().getEncodedGuidePosts(); onChanged(); return this; } @@ -819,12 +975,13 @@ public Builder clearGuidePostsCount() { descriptor; static { java.lang.String[] descriptorData = { - "\n\021PGuidePosts.proto\"r\n\013PGuidePosts\022\022\n\ngu" + - "idePosts\030\001 \001(\014\022\021\n\tbyteCount\030\002 \001(\003\022\020\n\010row" + - "Count\030\003 \001(\003\022\021\n\tmaxLength\030\004 \001(\005\022\027\n\017guideP" + - "ostsCount\030\005 \001(\005BE\n(org.apache.phoenix.co" + - "processor.generatedB\021PGuidePostsProtosH\001" + - "\210\001\001\240\001\001" + "\n\021PGuidePosts.proto\"\224\001\n\013PGuidePosts\022\022\n\ng" + + "uidePosts\030\001 \003(\014\022\021\n\tbyteCount\030\002 \001(\003\022\020\n\010ro" + + "wCount\030\003 \001(\003\022\021\n\tmaxLength\030\004 \001(\005\022\036\n\026encod" + + "edGuidePostsCount\030\005 \001(\005\022\031\n\021encodedGuideP" + + "osts\030\006 \001(\014BE\n(org.apache.phoenix.coproce" + + "ssor.generatedB\021PGuidePostsProtosH\001\210\001\001\240\001" + + "\001" }; com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner = new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() { @@ -836,7 +993,7 @@ public com.google.protobuf.ExtensionRegistry assignDescriptors( internal_static_PGuidePosts_fieldAccessorTable = new com.google.protobuf.GeneratedMessage.FieldAccessorTable( internal_static_PGuidePosts_descriptor, - new java.lang.String[] { "GuidePosts", "ByteCount", "RowCount", "MaxLength", "GuidePostsCount", }); + new java.lang.String[] { "GuidePosts", "ByteCount", "RowCount", "MaxLength", "EncodedGuidePostsCount", "EncodedGuidePosts", }); return null; } }; diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/PTableProtos.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/PTableProtos.java index 04f8849cfda..479aa880d4a 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/PTableProtos.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/PTableProtos.java @@ -1901,15 +1901,15 @@ public interface PTableStatsOrBuilder */ org.apache.phoenix.coprocessor.generated.PGuidePostsProtos.PGuidePostsOrBuilder getPGuidePostsOrBuilder(); - // optional bytes guidePosts = 7; + // optional bytes encodedGuidePosts = 7; /** - * optional bytes guidePosts = 7; + * optional bytes encodedGuidePosts = 7; */ - boolean hasGuidePosts(); + boolean hasEncodedGuidePosts(); /** - * optional bytes guidePosts = 7; + * optional bytes encodedGuidePosts = 7; */ - com.google.protobuf.ByteString getGuidePosts(); + com.google.protobuf.ByteString getEncodedGuidePosts(); // optional int32 maxLength = 8; /** @@ -2015,7 +2015,7 @@ private PTableStats( } case 58: { bitField0_ |= 0x00000020; - guidePosts_ = input.readBytes(); + encodedGuidePosts_ = input.readBytes(); break; } case 64: { @@ -2175,20 +2175,20 @@ public org.apache.phoenix.coprocessor.generated.PGuidePostsProtos.PGuidePostsOrB return pGuidePosts_; } - // optional bytes guidePosts = 7; - public static final int GUIDEPOSTS_FIELD_NUMBER = 7; - private com.google.protobuf.ByteString guidePosts_; + // optional bytes encodedGuidePosts = 7; + public static final int ENCODEDGUIDEPOSTS_FIELD_NUMBER = 7; + private com.google.protobuf.ByteString encodedGuidePosts_; /** - * optional bytes guidePosts = 7; + * optional bytes encodedGuidePosts = 7; */ - public boolean hasGuidePosts() { + public boolean hasEncodedGuidePosts() { return ((bitField0_ & 0x00000020) == 0x00000020); } /** - * optional bytes guidePosts = 7; + * optional bytes encodedGuidePosts = 7; */ - public com.google.protobuf.ByteString getGuidePosts() { - return guidePosts_; + public com.google.protobuf.ByteString getEncodedGuidePosts() { + return encodedGuidePosts_; } // optional int32 maxLength = 8; @@ -2214,7 +2214,7 @@ private void initFields() { keyBytesCount_ = 0L; guidePostsCount_ = 0; pGuidePosts_ = org.apache.phoenix.coprocessor.generated.PGuidePostsProtos.PGuidePosts.getDefaultInstance(); - guidePosts_ = com.google.protobuf.ByteString.EMPTY; + encodedGuidePosts_ = com.google.protobuf.ByteString.EMPTY; maxLength_ = 0; } private byte memoizedIsInitialized = -1; @@ -2252,7 +2252,7 @@ public void writeTo(com.google.protobuf.CodedOutputStream output) output.writeMessage(6, pGuidePosts_); } if (((bitField0_ & 0x00000020) == 0x00000020)) { - output.writeBytes(7, guidePosts_); + output.writeBytes(7, encodedGuidePosts_); } if (((bitField0_ & 0x00000040) == 0x00000040)) { output.writeInt32(8, maxLength_); @@ -2297,7 +2297,7 @@ public int getSerializedSize() { } if (((bitField0_ & 0x00000020) == 0x00000020)) { size += com.google.protobuf.CodedOutputStream - .computeBytesSize(7, guidePosts_); + .computeBytesSize(7, encodedGuidePosts_); } if (((bitField0_ & 0x00000040) == 0x00000040)) { size += com.google.protobuf.CodedOutputStream @@ -2353,10 +2353,10 @@ public boolean equals(final java.lang.Object obj) { result = result && getPGuidePosts() .equals(other.getPGuidePosts()); } - result = result && (hasGuidePosts() == other.hasGuidePosts()); - if (hasGuidePosts()) { - result = result && getGuidePosts() - .equals(other.getGuidePosts()); + result = result && (hasEncodedGuidePosts() == other.hasEncodedGuidePosts()); + if (hasEncodedGuidePosts()) { + result = result && getEncodedGuidePosts() + .equals(other.getEncodedGuidePosts()); } result = result && (hasMaxLength() == other.hasMaxLength()); if (hasMaxLength()) { @@ -2400,9 +2400,9 @@ public int hashCode() { hash = (37 * hash) + PGUIDEPOSTS_FIELD_NUMBER; hash = (53 * hash) + getPGuidePosts().hashCode(); } - if (hasGuidePosts()) { - hash = (37 * hash) + GUIDEPOSTS_FIELD_NUMBER; - hash = (53 * hash) + getGuidePosts().hashCode(); + if (hasEncodedGuidePosts()) { + hash = (37 * hash) + ENCODEDGUIDEPOSTS_FIELD_NUMBER; + hash = (53 * hash) + getEncodedGuidePosts().hashCode(); } if (hasMaxLength()) { hash = (37 * hash) + MAXLENGTH_FIELD_NUMBER; @@ -2534,7 +2534,7 @@ public Builder clear() { pGuidePostsBuilder_.clear(); } bitField0_ = (bitField0_ & ~0x00000020); - guidePosts_ = com.google.protobuf.ByteString.EMPTY; + encodedGuidePosts_ = com.google.protobuf.ByteString.EMPTY; bitField0_ = (bitField0_ & ~0x00000040); maxLength_ = 0; bitField0_ = (bitField0_ & ~0x00000080); @@ -2598,7 +2598,7 @@ public org.apache.phoenix.coprocessor.generated.PTableProtos.PTableStats buildPa if (((from_bitField0_ & 0x00000040) == 0x00000040)) { to_bitField0_ |= 0x00000020; } - result.guidePosts_ = guidePosts_; + result.encodedGuidePosts_ = encodedGuidePosts_; if (((from_bitField0_ & 0x00000080) == 0x00000080)) { to_bitField0_ |= 0x00000040; } @@ -2644,8 +2644,8 @@ public Builder mergeFrom(org.apache.phoenix.coprocessor.generated.PTableProtos.P if (other.hasPGuidePosts()) { mergePGuidePosts(other.getPGuidePosts()); } - if (other.hasGuidePosts()) { - setGuidePosts(other.getGuidePosts()); + if (other.hasEncodedGuidePosts()) { + setEncodedGuidePosts(other.getEncodedGuidePosts()); } if (other.hasMaxLength()) { setMaxLength(other.getMaxLength()); @@ -3005,38 +3005,38 @@ public org.apache.phoenix.coprocessor.generated.PGuidePostsProtos.PGuidePostsOrB return pGuidePostsBuilder_; } - // optional bytes guidePosts = 7; - private com.google.protobuf.ByteString guidePosts_ = com.google.protobuf.ByteString.EMPTY; + // optional bytes encodedGuidePosts = 7; + private com.google.protobuf.ByteString encodedGuidePosts_ = com.google.protobuf.ByteString.EMPTY; /** - * optional bytes guidePosts = 7; + * optional bytes encodedGuidePosts = 7; */ - public boolean hasGuidePosts() { + public boolean hasEncodedGuidePosts() { return ((bitField0_ & 0x00000040) == 0x00000040); } /** - * optional bytes guidePosts = 7; + * optional bytes encodedGuidePosts = 7; */ - public com.google.protobuf.ByteString getGuidePosts() { - return guidePosts_; + public com.google.protobuf.ByteString getEncodedGuidePosts() { + return encodedGuidePosts_; } /** - * optional bytes guidePosts = 7; + * optional bytes encodedGuidePosts = 7; */ - public Builder setGuidePosts(com.google.protobuf.ByteString value) { + public Builder setEncodedGuidePosts(com.google.protobuf.ByteString value) { if (value == null) { throw new NullPointerException(); } bitField0_ |= 0x00000040; - guidePosts_ = value; + encodedGuidePosts_ = value; onChanged(); return this; } /** - * optional bytes guidePosts = 7; + * optional bytes encodedGuidePosts = 7; */ - public Builder clearGuidePosts() { + public Builder clearEncodedGuidePosts() { bitField0_ = (bitField0_ & ~0x00000040); - guidePosts_ = getDefaultInstance().getGuidePosts(); + encodedGuidePosts_ = getDefaultInstance().getEncodedGuidePosts(); onChanged(); return this; } @@ -7009,32 +7009,32 @@ public Builder clearTransactional() { "\006 \002(\010\022\020\n\010position\030\007 \002(\005\022\021\n\tsortOrder\030\010 \002" + "(\005\022\021\n\tarraySize\030\t \001(\005\022\024\n\014viewConstant\030\n " + "\001(\014\022\026\n\016viewReferenced\030\013 \001(\010\022\022\n\nexpressio" + - "n\030\014 \001(\t\022\026\n\016isRowTimestamp\030\r \001(\010\"\301\001\n\013PTab" + + "n\030\014 \001(\t\022\026\n\016isRowTimestamp\030\r \001(\010\"\310\001\n\013PTab" + "leStats\022\013\n\003key\030\001 \002(\014\022\016\n\006values\030\002 \003(\014\022\033\n\023" + "guidePostsByteCount\030\003 \001(\003\022\025\n\rkeyBytesCou", "nt\030\004 \001(\003\022\027\n\017guidePostsCount\030\005 \001(\005\022!\n\013pGu" + - "idePosts\030\006 \001(\0132\014.PGuidePosts\022\022\n\nguidePos" + - "ts\030\007 \001(\014\022\021\n\tmaxLength\030\010 \001(\005\"\206\005\n\006PTable\022\027" + - "\n\017schemaNameBytes\030\001 \002(\014\022\026\n\016tableNameByte" + - "s\030\002 \002(\014\022\036\n\ttableType\030\003 \002(\0162\013.PTableType\022" + - "\022\n\nindexState\030\004 \001(\t\022\026\n\016sequenceNumber\030\005 " + - "\002(\003\022\021\n\ttimeStamp\030\006 \002(\003\022\023\n\013pkNameBytes\030\007 " + - "\001(\014\022\021\n\tbucketNum\030\010 \002(\005\022\031\n\007columns\030\t \003(\0132" + - "\010.PColumn\022\030\n\007indexes\030\n \003(\0132\007.PTable\022\027\n\017i" + - "sImmutableRows\030\013 \002(\010\022 \n\nguidePosts\030\014 \003(\013", - "2\014.PTableStats\022\032\n\022dataTableNameBytes\030\r \001" + - "(\014\022\031\n\021defaultFamilyName\030\016 \001(\014\022\022\n\ndisable" + - "WAL\030\017 \002(\010\022\023\n\013multiTenant\030\020 \002(\010\022\020\n\010viewTy" + - "pe\030\021 \001(\014\022\025\n\rviewStatement\030\022 \001(\014\022\025\n\rphysi" + - "calNames\030\023 \003(\014\022\020\n\010tenantId\030\024 \001(\014\022\023\n\013view" + - "IndexId\030\025 \001(\005\022\021\n\tindexType\030\026 \001(\014\022\026\n\016stat" + - "sTimeStamp\030\027 \001(\003\022\022\n\nstoreNulls\030\030 \001(\010\022\027\n\017" + - "baseColumnCount\030\031 \001(\005\022\036\n\026rowKeyOrderOpti" + - "mizable\030\032 \001(\010\022\025\n\rtransactional\030\033 \001(\010*A\n\n" + - "PTableType\022\n\n\006SYSTEM\020\000\022\010\n\004USER\020\001\022\010\n\004VIEW", - "\020\002\022\t\n\005INDEX\020\003\022\010\n\004JOIN\020\004B@\n(org.apache.ph" + - "oenix.coprocessor.generatedB\014PTableProto" + - "sH\001\210\001\001\240\001\001" + "idePosts\030\006 \001(\0132\014.PGuidePosts\022\031\n\021encodedG" + + "uidePosts\030\007 \001(\014\022\021\n\tmaxLength\030\010 \001(\005\"\206\005\n\006P" + + "Table\022\027\n\017schemaNameBytes\030\001 \002(\014\022\026\n\016tableN" + + "ameBytes\030\002 \002(\014\022\036\n\ttableType\030\003 \002(\0162\013.PTab" + + "leType\022\022\n\nindexState\030\004 \001(\t\022\026\n\016sequenceNu" + + "mber\030\005 \002(\003\022\021\n\ttimeStamp\030\006 \002(\003\022\023\n\013pkNameB" + + "ytes\030\007 \001(\014\022\021\n\tbucketNum\030\010 \002(\005\022\031\n\007columns" + + "\030\t \003(\0132\010.PColumn\022\030\n\007indexes\030\n \003(\0132\007.PTab" + + "le\022\027\n\017isImmutableRows\030\013 \002(\010\022 \n\nguidePost", + "s\030\014 \003(\0132\014.PTableStats\022\032\n\022dataTableNameBy" + + "tes\030\r \001(\014\022\031\n\021defaultFamilyName\030\016 \001(\014\022\022\n\n" + + "disableWAL\030\017 \002(\010\022\023\n\013multiTenant\030\020 \002(\010\022\020\n" + + "\010viewType\030\021 \001(\014\022\025\n\rviewStatement\030\022 \001(\014\022\025" + + "\n\rphysicalNames\030\023 \003(\014\022\020\n\010tenantId\030\024 \001(\014\022" + + "\023\n\013viewIndexId\030\025 \001(\005\022\021\n\tindexType\030\026 \001(\014\022" + + "\026\n\016statsTimeStamp\030\027 \001(\003\022\022\n\nstoreNulls\030\030 " + + "\001(\010\022\027\n\017baseColumnCount\030\031 \001(\005\022\036\n\026rowKeyOr" + + "derOptimizable\030\032 \001(\010\022\025\n\rtransactional\030\033 " + + "\001(\010*A\n\nPTableType\022\n\n\006SYSTEM\020\000\022\010\n\004USER\020\001\022", + "\010\n\004VIEW\020\002\022\t\n\005INDEX\020\003\022\010\n\004JOIN\020\004B@\n(org.ap" + + "ache.phoenix.coprocessor.generatedB\014PTab" + + "leProtosH\001\210\001\001\240\001\001" }; com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner = new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() { @@ -7052,7 +7052,7 @@ public com.google.protobuf.ExtensionRegistry assignDescriptors( internal_static_PTableStats_fieldAccessorTable = new com.google.protobuf.GeneratedMessage.FieldAccessorTable( internal_static_PTableStats_descriptor, - new java.lang.String[] { "Key", "Values", "GuidePostsByteCount", "KeyBytesCount", "GuidePostsCount", "PGuidePosts", "GuidePosts", "MaxLength", }); + new java.lang.String[] { "Key", "Values", "GuidePostsByteCount", "KeyBytesCount", "GuidePostsCount", "PGuidePosts", "EncodedGuidePosts", "MaxLength", }); internal_static_PTable_descriptor = getDescriptor().getMessageTypes().get(2); internal_static_PTable_fieldAccessorTable = new diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java index 90ddd786dba..c99c48514cc 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java @@ -1008,9 +1008,9 @@ public static PTable createFromProto(PTableProtos.PTable table) { long guidePostsByteCount = pGuidePosts.getByteCount(); long rowCount = pGuidePosts.getRowCount(); int maxLength = pGuidePosts.getMaxLength(); - int guidePostsCount = pGuidePosts.getGuidePostsCount(); + int guidePostsCount = pGuidePosts.getEncodedGuidePostsCount(); GuidePostsInfo info = new GuidePostsInfo(guidePostsByteCount, - new ImmutableBytesWritable(HBaseZeroCopyByteString.zeroCopyGetBytes(pGuidePosts.getGuidePosts())), rowCount, maxLength, guidePostsCount); + new ImmutableBytesWritable(HBaseZeroCopyByteString.zeroCopyGetBytes(pGuidePosts.getEncodedGuidePosts())), rowCount, maxLength, guidePostsCount); tableGuidePosts.put(pTableStatsProto.getKey().toByteArray(), info); } PTableStats stats = new PTableStatsImpl(tableGuidePosts, table.getStatsTimeStamp()); @@ -1112,16 +1112,16 @@ public static PTableProtos.PTable toProto(PTable table) { for (Map.Entry entry : table.getTableStats().getGuidePosts().entrySet()) { PTableProtos.PTableStats.Builder statsBuilder = PTableProtos.PTableStats.newBuilder(); statsBuilder.setKey(ByteStringer.wrap(entry.getKey())); - statsBuilder.setGuidePosts(ByteStringer.wrap(entry.getValue().getGuidePosts().get())); + statsBuilder.setEncodedGuidePosts(ByteStringer.wrap(entry.getValue().getGuidePosts().get())); statsBuilder.setGuidePostsByteCount(entry.getValue().getByteCount()); statsBuilder.setMaxLength(entry.getValue().getMaxLength()); statsBuilder.setGuidePostsCount(entry.getValue().getGuidePostsCount()); PGuidePostsProtos.PGuidePosts.Builder guidePstsBuilder = PGuidePostsProtos.PGuidePosts.newBuilder(); - guidePstsBuilder.setGuidePosts(ByteStringer.wrap(entry.getValue().getGuidePosts().get())); + guidePstsBuilder.setEncodedGuidePosts(ByteStringer.wrap(entry.getValue().getGuidePosts().get())); guidePstsBuilder.setByteCount(entry.getValue().getByteCount()); guidePstsBuilder.setRowCount(entry.getValue().getRowCount()); guidePstsBuilder.setMaxLength(entry.getValue().getMaxLength()); - guidePstsBuilder.setGuidePostsCount(entry.getValue().getGuidePostsCount()); + guidePstsBuilder.setEncodedGuidePostsCount(entry.getValue().getGuidePostsCount()); statsBuilder.setPGuidePosts(guidePstsBuilder); builder.addGuidePosts(statsBuilder.build()); } diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/UpgradeUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/UpgradeUtil.java index c9318512975..4d6afcea50d 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/util/UpgradeUtil.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/util/UpgradeUtil.java @@ -52,12 +52,14 @@ import javax.annotation.Nonnull; import javax.annotation.Nullable; +import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.Delete; +import org.apache.hadoop.hbase.client.Get; import org.apache.hadoop.hbase.client.HBaseAdmin; import org.apache.hadoop.hbase.client.HTableInterface; import org.apache.hadoop.hbase.client.Mutation; @@ -100,6 +102,7 @@ public class UpgradeUtil { private static final Logger logger = LoggerFactory.getLogger(UpgradeUtil.class); private static final byte[] SEQ_PREFIX_BYTES = ByteUtil.concat(QueryConstants.SEPARATOR_BYTE_ARRAY, Bytes.toBytes("_SEQ_")); + public static final byte[] UPGRADE_TO_4_7_COLUMN_NAME = Bytes.toBytes("UPGRADE_TO_4_7"); public static String UPSERT_BASE_COLUMN_COUNT_IN_HEADER_ROW = "UPSERT " + "INTO SYSTEM.CATALOG " @@ -1215,4 +1218,67 @@ public static void addRowKeyOrderOptimizableCell(List tableMetadata, b MetaDataEndpointImpl.ROW_KEY_ORDER_OPTIMIZABLE_BYTES, PBoolean.INSTANCE.toBytes(true)); tableMetadata.add(put); } + + public static boolean truncateStats(HTableInterface metaTable, HTableInterface statsTable) + throws IOException, InterruptedException { + List columnCells = metaTable + .get(new Get(SchemaUtil.getTableKey(null, PhoenixDatabaseMetaData.SYSTEM_SCHEMA_NAME, + PhoenixDatabaseMetaData.SYSTEM_CATALOG_TABLE))) + .getColumnCells(PhoenixDatabaseMetaData.TABLE_FAMILY_BYTES, QueryConstants.EMPTY_COLUMN_BYTES); + if (!columnCells.isEmpty() + && columnCells.get(0).getTimestamp() < MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_7_0) { + + byte[] statsTableKey = SchemaUtil.getTableKey(null, PhoenixDatabaseMetaData.SYSTEM_SCHEMA_NAME, + PhoenixDatabaseMetaData.SYSTEM_STATS_TABLE); + KeyValue upgradeKV = KeyValueUtil.newKeyValue(statsTableKey, PhoenixDatabaseMetaData.TABLE_FAMILY_BYTES, + UPGRADE_TO_4_7_COLUMN_NAME, MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_7_0 - 1, + ByteUtil.EMPTY_BYTE_ARRAY); + Put upgradePut = new Put(statsTableKey); + upgradePut.add(upgradeKV); + + // check for null in UPGRADE_TO_4_7_COLUMN_NAME in checkAndPut so that only single client + // drop the rows of SYSTEM.STATS + if (metaTable.checkAndPut(statsTableKey, PhoenixDatabaseMetaData.TABLE_FAMILY_BYTES, + UPGRADE_TO_4_7_COLUMN_NAME, null, upgradePut)) { + List mutations = Lists.newArrayListWithExpectedSize(1000); + Scan scan = new Scan(); + scan.setRaw(true); + scan.setMaxVersions(); + ResultScanner statsScanner = statsTable.getScanner(scan); + Result r; + mutations.clear(); + int count = 0; + while ((r = statsScanner.next()) != null) { + Delete delete = null; + for (KeyValue keyValue : r.raw()) { + if (KeyValue.Type.codeToType(keyValue.getType()) == KeyValue.Type.Put) { + if (delete == null) { + delete = new Delete(keyValue.getRow()); + } + KeyValue deleteKeyValue = new KeyValue(keyValue.getRowArray(), keyValue.getRowOffset(), + keyValue.getRowLength(), keyValue.getFamilyArray(), keyValue.getFamilyOffset(), + keyValue.getFamilyLength(), keyValue.getQualifierArray(), + keyValue.getQualifierOffset(), keyValue.getQualifierLength(), + keyValue.getTimestamp(), KeyValue.Type.Delete, ByteUtil.EMPTY_BYTE_ARRAY, 0, 0); + delete.addDeleteMarker(deleteKeyValue); + } + } + if (delete != null) { + mutations.add(delete); + if (count > 10) { + statsTable.batch(mutations); + mutations.clear(); + count = 0; + } + count++; + } + } + if (!mutations.isEmpty()) { + statsTable.batch(mutations); + } + return true; + } + } + return false; + } } diff --git a/phoenix-protocol/src/main/PGuidePosts.proto b/phoenix-protocol/src/main/PGuidePosts.proto index d0c7c9bdfd7..14de2ebc9e1 100644 --- a/phoenix-protocol/src/main/PGuidePosts.proto +++ b/phoenix-protocol/src/main/PGuidePosts.proto @@ -24,9 +24,10 @@ option java_generic_services = true; option java_generate_equals_and_hash = true; option optimize_for = SPEED; message PGuidePosts { - optional bytes guidePosts = 1; + repeated bytes guidePosts = 1; optional int64 byteCount = 2; optional int64 rowCount = 3; optional int32 maxLength = 4; - optional int32 guidePostsCount = 5; + optional int32 encodedGuidePostsCount = 5; + optional bytes encodedGuidePosts = 6; } \ No newline at end of file diff --git a/phoenix-protocol/src/main/PTable.proto b/phoenix-protocol/src/main/PTable.proto index 59f34449141..1f83aa7edf2 100644 --- a/phoenix-protocol/src/main/PTable.proto +++ b/phoenix-protocol/src/main/PTable.proto @@ -57,7 +57,7 @@ message PTableStats { optional int64 keyBytesCount = 4; optional int32 guidePostsCount = 5; optional PGuidePosts pGuidePosts = 6; - optional bytes guidePosts = 7; + optional bytes encodedGuidePosts = 7; optional int32 maxLength = 8; } From 785c687f68ca6f913528f0cb1bb050dd5655a739 Mon Sep 17 00:00:00 2001 From: Ankit Singhal Date: Fri, 15 Jan 2016 16:00:23 +0530 Subject: [PATCH 33/47] PHOENIX-2417 Compress memory used by row key byte[] of guideposts --- .../apache/phoenix/compile/ScanRanges.java | 10 +- .../UngroupedAggregateRegionObserver.java | 1 + .../generated/PGuidePostsProtos.java | 370 +++-- .../coprocessor/generated/PTableProtos.java | 327 +++-- .../generated/StatCollectorProtos.java | 1269 ----------------- .../org/apache/phoenix/execute/ScanPlan.java | 2 +- .../phoenix/iterate/BaseResultIterators.java | 144 +- .../org/apache/phoenix/schema/PTableImpl.java | 32 +- .../phoenix/schema/stats/GuidePostsInfo.java | 149 +- .../phoenix/schema/stats/PTableStatsImpl.java | 57 +- .../schema/stats/StatisticsCollector.java | 33 +- .../schema/stats/StatisticsScanner.java | 4 +- .../phoenix/schema/stats/StatisticsUtil.java | 28 +- .../schema/stats/StatisticsWriter.java | 49 +- .../org/apache/phoenix/util/ByteUtil.java | 2 + .../org/apache/phoenix/util/CodecUtils.java | 89 ++ .../phoenix/util/PrefixByteDecoder.java | 85 ++ .../phoenix/util/PrefixByteEncoder.java | 99 ++ .../phoenix/filter/SkipScanBigFilterTest.java | 10 +- .../util/PrefixByteEncoderDecoderTest.java | 96 ++ phoenix-protocol/src/main/PGuidePosts.proto | 4 +- phoenix-protocol/src/main/PTable.proto | 3 +- 22 files changed, 1133 insertions(+), 1730 deletions(-) delete mode 100644 phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/StatCollectorProtos.java create mode 100644 phoenix-core/src/main/java/org/apache/phoenix/util/CodecUtils.java create mode 100644 phoenix-core/src/main/java/org/apache/phoenix/util/PrefixByteDecoder.java create mode 100644 phoenix-core/src/main/java/org/apache/phoenix/util/PrefixByteEncoder.java create mode 100644 phoenix-core/src/test/java/org/apache/phoenix/util/PrefixByteEncoderDecoderTest.java diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/ScanRanges.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/ScanRanges.java index 4d343f342e8..91d07ac898d 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/compile/ScanRanges.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/ScanRanges.java @@ -237,7 +237,9 @@ private static byte[] stripPrefix(byte[] key, int keyOffset) { return temp; } - public Scan intersectScan(Scan scan, final byte[] originalStartKey, final byte[] originalStopKey, final int keyOffset, boolean crossesRegionBoundary) { + public Scan intersectScan(Scan scan, final ImmutableBytesWritable originalStartKeyPtr, final ImmutableBytesWritable originalStopKeyPtr, final int keyOffset, boolean crossesRegionBoundary) { + byte[] originalStartKey= originalStartKeyPtr.get(); + byte[] originalStopKey= originalStopKeyPtr.get(); byte[] startKey = originalStartKey; byte[] stopKey = originalStopKey; if (stopKey.length > 0 && Bytes.compareTo(startKey, stopKey) >= 0) { @@ -384,8 +386,8 @@ public Scan intersectScan(Scan scan, final byte[] originalStartKey, final byte[] if (scanStopKey.length > 0 && Bytes.compareTo(scanStartKey, scanStopKey) >= 0) { return null; } - newScan.setStartRow(scanStartKey); - newScan.setStopRow(scanStopKey); + newScan.setStartRow(Bytes.copy(scanStartKey)); + newScan.setStopRow(Bytes.copy(scanStopKey)); if(keyOffset > 0) { newScan.setAttribute(STARTKEY_OFFSET, Bytes.toBytes(keyOffset)); } @@ -416,7 +418,7 @@ public boolean intersects(byte[] lowerInclusiveKey, byte[] upperExclusiveKey, in } //return filter.hasIntersect(lowerInclusiveKey, upperExclusiveKey); - return intersectScan(null, lowerInclusiveKey, upperExclusiveKey, keyOffset, crossesRegionBoundary) == HAS_INTERSECTION; + return intersectScan(null, new ImmutableBytesWritable(lowerInclusiveKey), new ImmutableBytesWritable(upperExclusiveKey), keyOffset, crossesRegionBoundary) == HAS_INTERSECTION; } public SkipScanFilter getSkipScanFilter() { diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java index f332e60b960..63869bb2851 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java @@ -783,6 +783,7 @@ private long collectStatsInternal() throws IOException { } finally { try { if (noErrors && !compactionRunning) { + stats.updateGuidePosts(); stats.updateStatistic(region); logger.info("UPDATE STATISTICS finished successfully for scanner: " + innerScanner + ". Number of rows scanned: " + rowCount diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/PGuidePostsProtos.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/PGuidePostsProtos.java index 9f965309b0e..58999740ed8 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/PGuidePostsProtos.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/PGuidePostsProtos.java @@ -11,19 +11,15 @@ public static void registerAllExtensions( public interface PGuidePostsOrBuilder extends com.google.protobuf.MessageOrBuilder { - // repeated bytes guidePosts = 1; + // optional bytes guidePosts = 1; /** - * repeated bytes guidePosts = 1; + * optional bytes guidePosts = 1; */ - java.util.List getGuidePostsList(); + boolean hasGuidePosts(); /** - * repeated bytes guidePosts = 1; + * optional bytes guidePosts = 1; */ - int getGuidePostsCount(); - /** - * repeated bytes guidePosts = 1; - */ - com.google.protobuf.ByteString getGuidePosts(int index); + com.google.protobuf.ByteString getGuidePosts(); // optional int64 byteCount = 2; /** @@ -44,6 +40,26 @@ public interface PGuidePostsOrBuilder * optional int64 rowCount = 3; */ long getRowCount(); + + // optional int32 maxLength = 4; + /** + * optional int32 maxLength = 4; + */ + boolean hasMaxLength(); + /** + * optional int32 maxLength = 4; + */ + int getMaxLength(); + + // optional int32 guidePostsCount = 5; + /** + * optional int32 guidePostsCount = 5; + */ + boolean hasGuidePostsCount(); + /** + * optional int32 guidePostsCount = 5; + */ + int getGuidePostsCount(); } /** * Protobuf type {@code PGuidePosts} @@ -97,23 +113,30 @@ private PGuidePosts( break; } case 10: { - if (!((mutable_bitField0_ & 0x00000001) == 0x00000001)) { - guidePosts_ = new java.util.ArrayList(); - mutable_bitField0_ |= 0x00000001; - } - guidePosts_.add(input.readBytes()); + bitField0_ |= 0x00000001; + guidePosts_ = input.readBytes(); break; } case 16: { - bitField0_ |= 0x00000001; + bitField0_ |= 0x00000002; byteCount_ = input.readInt64(); break; } case 24: { - bitField0_ |= 0x00000002; + bitField0_ |= 0x00000004; rowCount_ = input.readInt64(); break; } + case 32: { + bitField0_ |= 0x00000008; + maxLength_ = input.readInt32(); + break; + } + case 40: { + bitField0_ |= 0x00000010; + guidePostsCount_ = input.readInt32(); + break; + } } } } catch (com.google.protobuf.InvalidProtocolBufferException e) { @@ -122,9 +145,6 @@ private PGuidePosts( throw new com.google.protobuf.InvalidProtocolBufferException( e.getMessage()).setUnfinishedMessage(this); } finally { - if (((mutable_bitField0_ & 0x00000001) == 0x00000001)) { - guidePosts_ = java.util.Collections.unmodifiableList(guidePosts_); - } this.unknownFields = unknownFields.build(); makeExtensionsImmutable(); } @@ -157,27 +177,20 @@ public com.google.protobuf.Parser getParserForType() { } private int bitField0_; - // repeated bytes guidePosts = 1; + // optional bytes guidePosts = 1; public static final int GUIDEPOSTS_FIELD_NUMBER = 1; - private java.util.List guidePosts_; - /** - * repeated bytes guidePosts = 1; - */ - public java.util.List - getGuidePostsList() { - return guidePosts_; - } + private com.google.protobuf.ByteString guidePosts_; /** - * repeated bytes guidePosts = 1; + * optional bytes guidePosts = 1; */ - public int getGuidePostsCount() { - return guidePosts_.size(); + public boolean hasGuidePosts() { + return ((bitField0_ & 0x00000001) == 0x00000001); } /** - * repeated bytes guidePosts = 1; + * optional bytes guidePosts = 1; */ - public com.google.protobuf.ByteString getGuidePosts(int index) { - return guidePosts_.get(index); + public com.google.protobuf.ByteString getGuidePosts() { + return guidePosts_; } // optional int64 byteCount = 2; @@ -187,7 +200,7 @@ public com.google.protobuf.ByteString getGuidePosts(int index) { * optional int64 byteCount = 2; */ public boolean hasByteCount() { - return ((bitField0_ & 0x00000001) == 0x00000001); + return ((bitField0_ & 0x00000002) == 0x00000002); } /** * optional int64 byteCount = 2; @@ -203,7 +216,7 @@ public long getByteCount() { * optional int64 rowCount = 3; */ public boolean hasRowCount() { - return ((bitField0_ & 0x00000002) == 0x00000002); + return ((bitField0_ & 0x00000004) == 0x00000004); } /** * optional int64 rowCount = 3; @@ -212,10 +225,44 @@ public long getRowCount() { return rowCount_; } + // optional int32 maxLength = 4; + public static final int MAXLENGTH_FIELD_NUMBER = 4; + private int maxLength_; + /** + * optional int32 maxLength = 4; + */ + public boolean hasMaxLength() { + return ((bitField0_ & 0x00000008) == 0x00000008); + } + /** + * optional int32 maxLength = 4; + */ + public int getMaxLength() { + return maxLength_; + } + + // optional int32 guidePostsCount = 5; + public static final int GUIDEPOSTSCOUNT_FIELD_NUMBER = 5; + private int guidePostsCount_; + /** + * optional int32 guidePostsCount = 5; + */ + public boolean hasGuidePostsCount() { + return ((bitField0_ & 0x00000010) == 0x00000010); + } + /** + * optional int32 guidePostsCount = 5; + */ + public int getGuidePostsCount() { + return guidePostsCount_; + } + private void initFields() { - guidePosts_ = java.util.Collections.emptyList(); + guidePosts_ = com.google.protobuf.ByteString.EMPTY; byteCount_ = 0L; rowCount_ = 0L; + maxLength_ = 0; + guidePostsCount_ = 0; } private byte memoizedIsInitialized = -1; public final boolean isInitialized() { @@ -229,15 +276,21 @@ public final boolean isInitialized() { public void writeTo(com.google.protobuf.CodedOutputStream output) throws java.io.IOException { getSerializedSize(); - for (int i = 0; i < guidePosts_.size(); i++) { - output.writeBytes(1, guidePosts_.get(i)); - } if (((bitField0_ & 0x00000001) == 0x00000001)) { - output.writeInt64(2, byteCount_); + output.writeBytes(1, guidePosts_); } if (((bitField0_ & 0x00000002) == 0x00000002)) { + output.writeInt64(2, byteCount_); + } + if (((bitField0_ & 0x00000004) == 0x00000004)) { output.writeInt64(3, rowCount_); } + if (((bitField0_ & 0x00000008) == 0x00000008)) { + output.writeInt32(4, maxLength_); + } + if (((bitField0_ & 0x00000010) == 0x00000010)) { + output.writeInt32(5, guidePostsCount_); + } getUnknownFields().writeTo(output); } @@ -247,23 +300,26 @@ public int getSerializedSize() { if (size != -1) return size; size = 0; - { - int dataSize = 0; - for (int i = 0; i < guidePosts_.size(); i++) { - dataSize += com.google.protobuf.CodedOutputStream - .computeBytesSizeNoTag(guidePosts_.get(i)); - } - size += dataSize; - size += 1 * getGuidePostsList().size(); - } if (((bitField0_ & 0x00000001) == 0x00000001)) { size += com.google.protobuf.CodedOutputStream - .computeInt64Size(2, byteCount_); + .computeBytesSize(1, guidePosts_); } if (((bitField0_ & 0x00000002) == 0x00000002)) { + size += com.google.protobuf.CodedOutputStream + .computeInt64Size(2, byteCount_); + } + if (((bitField0_ & 0x00000004) == 0x00000004)) { size += com.google.protobuf.CodedOutputStream .computeInt64Size(3, rowCount_); } + if (((bitField0_ & 0x00000008) == 0x00000008)) { + size += com.google.protobuf.CodedOutputStream + .computeInt32Size(4, maxLength_); + } + if (((bitField0_ & 0x00000010) == 0x00000010)) { + size += com.google.protobuf.CodedOutputStream + .computeInt32Size(5, guidePostsCount_); + } size += getUnknownFields().getSerializedSize(); memoizedSerializedSize = size; return size; @@ -287,8 +343,11 @@ public boolean equals(final java.lang.Object obj) { org.apache.phoenix.coprocessor.generated.PGuidePostsProtos.PGuidePosts other = (org.apache.phoenix.coprocessor.generated.PGuidePostsProtos.PGuidePosts) obj; boolean result = true; - result = result && getGuidePostsList() - .equals(other.getGuidePostsList()); + result = result && (hasGuidePosts() == other.hasGuidePosts()); + if (hasGuidePosts()) { + result = result && getGuidePosts() + .equals(other.getGuidePosts()); + } result = result && (hasByteCount() == other.hasByteCount()); if (hasByteCount()) { result = result && (getByteCount() @@ -299,6 +358,16 @@ public boolean equals(final java.lang.Object obj) { result = result && (getRowCount() == other.getRowCount()); } + result = result && (hasMaxLength() == other.hasMaxLength()); + if (hasMaxLength()) { + result = result && (getMaxLength() + == other.getMaxLength()); + } + result = result && (hasGuidePostsCount() == other.hasGuidePostsCount()); + if (hasGuidePostsCount()) { + result = result && (getGuidePostsCount() + == other.getGuidePostsCount()); + } result = result && getUnknownFields().equals(other.getUnknownFields()); return result; @@ -312,9 +381,9 @@ public int hashCode() { } int hash = 41; hash = (19 * hash) + getDescriptorForType().hashCode(); - if (getGuidePostsCount() > 0) { + if (hasGuidePosts()) { hash = (37 * hash) + GUIDEPOSTS_FIELD_NUMBER; - hash = (53 * hash) + getGuidePostsList().hashCode(); + hash = (53 * hash) + getGuidePosts().hashCode(); } if (hasByteCount()) { hash = (37 * hash) + BYTECOUNT_FIELD_NUMBER; @@ -324,6 +393,14 @@ public int hashCode() { hash = (37 * hash) + ROWCOUNT_FIELD_NUMBER; hash = (53 * hash) + hashLong(getRowCount()); } + if (hasMaxLength()) { + hash = (37 * hash) + MAXLENGTH_FIELD_NUMBER; + hash = (53 * hash) + getMaxLength(); + } + if (hasGuidePostsCount()) { + hash = (37 * hash) + GUIDEPOSTSCOUNT_FIELD_NUMBER; + hash = (53 * hash) + getGuidePostsCount(); + } hash = (29 * hash) + getUnknownFields().hashCode(); memoizedHashCode = hash; return hash; @@ -433,12 +510,16 @@ private static Builder create() { public Builder clear() { super.clear(); - guidePosts_ = java.util.Collections.emptyList(); + guidePosts_ = com.google.protobuf.ByteString.EMPTY; bitField0_ = (bitField0_ & ~0x00000001); byteCount_ = 0L; bitField0_ = (bitField0_ & ~0x00000002); rowCount_ = 0L; bitField0_ = (bitField0_ & ~0x00000004); + maxLength_ = 0; + bitField0_ = (bitField0_ & ~0x00000008); + guidePostsCount_ = 0; + bitField0_ = (bitField0_ & ~0x00000010); return this; } @@ -467,19 +548,26 @@ public org.apache.phoenix.coprocessor.generated.PGuidePostsProtos.PGuidePosts bu org.apache.phoenix.coprocessor.generated.PGuidePostsProtos.PGuidePosts result = new org.apache.phoenix.coprocessor.generated.PGuidePostsProtos.PGuidePosts(this); int from_bitField0_ = bitField0_; int to_bitField0_ = 0; - if (((bitField0_ & 0x00000001) == 0x00000001)) { - guidePosts_ = java.util.Collections.unmodifiableList(guidePosts_); - bitField0_ = (bitField0_ & ~0x00000001); + if (((from_bitField0_ & 0x00000001) == 0x00000001)) { + to_bitField0_ |= 0x00000001; } result.guidePosts_ = guidePosts_; if (((from_bitField0_ & 0x00000002) == 0x00000002)) { - to_bitField0_ |= 0x00000001; + to_bitField0_ |= 0x00000002; } result.byteCount_ = byteCount_; if (((from_bitField0_ & 0x00000004) == 0x00000004)) { - to_bitField0_ |= 0x00000002; + to_bitField0_ |= 0x00000004; } result.rowCount_ = rowCount_; + if (((from_bitField0_ & 0x00000008) == 0x00000008)) { + to_bitField0_ |= 0x00000008; + } + result.maxLength_ = maxLength_; + if (((from_bitField0_ & 0x00000010) == 0x00000010)) { + to_bitField0_ |= 0x00000010; + } + result.guidePostsCount_ = guidePostsCount_; result.bitField0_ = to_bitField0_; onBuilt(); return result; @@ -496,15 +584,8 @@ public Builder mergeFrom(com.google.protobuf.Message other) { public Builder mergeFrom(org.apache.phoenix.coprocessor.generated.PGuidePostsProtos.PGuidePosts other) { if (other == org.apache.phoenix.coprocessor.generated.PGuidePostsProtos.PGuidePosts.getDefaultInstance()) return this; - if (!other.guidePosts_.isEmpty()) { - if (guidePosts_.isEmpty()) { - guidePosts_ = other.guidePosts_; - bitField0_ = (bitField0_ & ~0x00000001); - } else { - ensureGuidePostsIsMutable(); - guidePosts_.addAll(other.guidePosts_); - } - onChanged(); + if (other.hasGuidePosts()) { + setGuidePosts(other.getGuidePosts()); } if (other.hasByteCount()) { setByteCount(other.getByteCount()); @@ -512,6 +593,12 @@ public Builder mergeFrom(org.apache.phoenix.coprocessor.generated.PGuidePostsPro if (other.hasRowCount()) { setRowCount(other.getRowCount()); } + if (other.hasMaxLength()) { + setMaxLength(other.getMaxLength()); + } + if (other.hasGuidePostsCount()) { + setGuidePostsCount(other.getGuidePostsCount()); + } this.mergeUnknownFields(other.getUnknownFields()); return this; } @@ -539,74 +626,38 @@ public Builder mergeFrom( } private int bitField0_; - // repeated bytes guidePosts = 1; - private java.util.List guidePosts_ = java.util.Collections.emptyList(); - private void ensureGuidePostsIsMutable() { - if (!((bitField0_ & 0x00000001) == 0x00000001)) { - guidePosts_ = new java.util.ArrayList(guidePosts_); - bitField0_ |= 0x00000001; - } - } + // optional bytes guidePosts = 1; + private com.google.protobuf.ByteString guidePosts_ = com.google.protobuf.ByteString.EMPTY; /** - * repeated bytes guidePosts = 1; + * optional bytes guidePosts = 1; */ - public java.util.List - getGuidePostsList() { - return java.util.Collections.unmodifiableList(guidePosts_); + public boolean hasGuidePosts() { + return ((bitField0_ & 0x00000001) == 0x00000001); } /** - * repeated bytes guidePosts = 1; + * optional bytes guidePosts = 1; */ - public int getGuidePostsCount() { - return guidePosts_.size(); + public com.google.protobuf.ByteString getGuidePosts() { + return guidePosts_; } /** - * repeated bytes guidePosts = 1; + * optional bytes guidePosts = 1; */ - public com.google.protobuf.ByteString getGuidePosts(int index) { - return guidePosts_.get(index); - } - /** - * repeated bytes guidePosts = 1; - */ - public Builder setGuidePosts( - int index, com.google.protobuf.ByteString value) { - if (value == null) { - throw new NullPointerException(); - } - ensureGuidePostsIsMutable(); - guidePosts_.set(index, value); - onChanged(); - return this; - } - /** - * repeated bytes guidePosts = 1; - */ - public Builder addGuidePosts(com.google.protobuf.ByteString value) { + public Builder setGuidePosts(com.google.protobuf.ByteString value) { if (value == null) { throw new NullPointerException(); } - ensureGuidePostsIsMutable(); - guidePosts_.add(value); - onChanged(); - return this; - } - /** - * repeated bytes guidePosts = 1; - */ - public Builder addAllGuidePosts( - java.lang.Iterable values) { - ensureGuidePostsIsMutable(); - super.addAll(values, guidePosts_); + bitField0_ |= 0x00000001; + guidePosts_ = value; onChanged(); return this; } /** - * repeated bytes guidePosts = 1; + * optional bytes guidePosts = 1; */ public Builder clearGuidePosts() { - guidePosts_ = java.util.Collections.emptyList(); bitField0_ = (bitField0_ & ~0x00000001); + guidePosts_ = getDefaultInstance().getGuidePosts(); onChanged(); return this; } @@ -677,6 +728,72 @@ public Builder clearRowCount() { return this; } + // optional int32 maxLength = 4; + private int maxLength_ ; + /** + * optional int32 maxLength = 4; + */ + public boolean hasMaxLength() { + return ((bitField0_ & 0x00000008) == 0x00000008); + } + /** + * optional int32 maxLength = 4; + */ + public int getMaxLength() { + return maxLength_; + } + /** + * optional int32 maxLength = 4; + */ + public Builder setMaxLength(int value) { + bitField0_ |= 0x00000008; + maxLength_ = value; + onChanged(); + return this; + } + /** + * optional int32 maxLength = 4; + */ + public Builder clearMaxLength() { + bitField0_ = (bitField0_ & ~0x00000008); + maxLength_ = 0; + onChanged(); + return this; + } + + // optional int32 guidePostsCount = 5; + private int guidePostsCount_ ; + /** + * optional int32 guidePostsCount = 5; + */ + public boolean hasGuidePostsCount() { + return ((bitField0_ & 0x00000010) == 0x00000010); + } + /** + * optional int32 guidePostsCount = 5; + */ + public int getGuidePostsCount() { + return guidePostsCount_; + } + /** + * optional int32 guidePostsCount = 5; + */ + public Builder setGuidePostsCount(int value) { + bitField0_ |= 0x00000010; + guidePostsCount_ = value; + onChanged(); + return this; + } + /** + * optional int32 guidePostsCount = 5; + */ + public Builder clearGuidePostsCount() { + bitField0_ = (bitField0_ & ~0x00000010); + guidePostsCount_ = 0; + onChanged(); + return this; + } + // @@protoc_insertion_point(builder_scope:PGuidePosts) } @@ -702,11 +819,12 @@ public Builder clearRowCount() { descriptor; static { java.lang.String[] descriptorData = { - "\n\021PGuidePosts.proto\"F\n\013PGuidePosts\022\022\n\ngu" + - "idePosts\030\001 \003(\014\022\021\n\tbyteCount\030\002 \001(\003\022\020\n\010row" + - "Count\030\003 \001(\003BE\n(org.apache.phoenix.coproc" + - "essor.generatedB\021PGuidePostsProtosH\001\210\001\001\240" + - "\001\001" + "\n\021PGuidePosts.proto\"r\n\013PGuidePosts\022\022\n\ngu" + + "idePosts\030\001 \001(\014\022\021\n\tbyteCount\030\002 \001(\003\022\020\n\010row" + + "Count\030\003 \001(\003\022\021\n\tmaxLength\030\004 \001(\005\022\027\n\017guideP" + + "ostsCount\030\005 \001(\005BE\n(org.apache.phoenix.co" + + "processor.generatedB\021PGuidePostsProtosH\001" + + "\210\001\001\240\001\001" }; com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner = new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() { @@ -718,7 +836,7 @@ public com.google.protobuf.ExtensionRegistry assignDescriptors( internal_static_PGuidePosts_fieldAccessorTable = new com.google.protobuf.GeneratedMessage.FieldAccessorTable( internal_static_PGuidePosts_descriptor, - new java.lang.String[] { "GuidePosts", "ByteCount", "RowCount", }); + new java.lang.String[] { "GuidePosts", "ByteCount", "RowCount", "MaxLength", "GuidePostsCount", }); return null; } }; diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/PTableProtos.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/PTableProtos.java index be8d7e2b93a..026d4def947 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/PTableProtos.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/PTableProtos.java @@ -1843,19 +1843,15 @@ public interface PTableStatsOrBuilder */ com.google.protobuf.ByteString getKey(); - // repeated bytes values = 2; + // optional bytes guidePosts = 2; /** - * repeated bytes values = 2; + * optional bytes guidePosts = 2; */ - java.util.List getValuesList(); + boolean hasGuidePosts(); /** - * repeated bytes values = 2; + * optional bytes guidePosts = 2; */ - int getValuesCount(); - /** - * repeated bytes values = 2; - */ - com.google.protobuf.ByteString getValues(int index); + com.google.protobuf.ByteString getGuidePosts(); // optional int64 guidePostsByteCount = 3; /** @@ -1900,6 +1896,16 @@ public interface PTableStatsOrBuilder * optional .PGuidePosts pGuidePosts = 6; */ org.apache.phoenix.coprocessor.generated.PGuidePostsProtos.PGuidePostsOrBuilder getPGuidePostsOrBuilder(); + + // optional int32 maxLength = 7; + /** + * optional int32 maxLength = 7; + */ + boolean hasMaxLength(); + /** + * optional int32 maxLength = 7; + */ + int getMaxLength(); } /** * Protobuf type {@code PTableStats} @@ -1958,31 +1964,28 @@ private PTableStats( break; } case 18: { - if (!((mutable_bitField0_ & 0x00000002) == 0x00000002)) { - values_ = new java.util.ArrayList(); - mutable_bitField0_ |= 0x00000002; - } - values_.add(input.readBytes()); + bitField0_ |= 0x00000002; + guidePosts_ = input.readBytes(); break; } case 24: { - bitField0_ |= 0x00000002; + bitField0_ |= 0x00000004; guidePostsByteCount_ = input.readInt64(); break; } case 32: { - bitField0_ |= 0x00000004; + bitField0_ |= 0x00000008; keyBytesCount_ = input.readInt64(); break; } case 40: { - bitField0_ |= 0x00000008; + bitField0_ |= 0x00000010; guidePostsCount_ = input.readInt32(); break; } case 50: { org.apache.phoenix.coprocessor.generated.PGuidePostsProtos.PGuidePosts.Builder subBuilder = null; - if (((bitField0_ & 0x00000010) == 0x00000010)) { + if (((bitField0_ & 0x00000020) == 0x00000020)) { subBuilder = pGuidePosts_.toBuilder(); } pGuidePosts_ = input.readMessage(org.apache.phoenix.coprocessor.generated.PGuidePostsProtos.PGuidePosts.PARSER, extensionRegistry); @@ -1990,7 +1993,12 @@ private PTableStats( subBuilder.mergeFrom(pGuidePosts_); pGuidePosts_ = subBuilder.buildPartial(); } - bitField0_ |= 0x00000010; + bitField0_ |= 0x00000020; + break; + } + case 56: { + bitField0_ |= 0x00000040; + maxLength_ = input.readInt32(); break; } } @@ -2001,9 +2009,6 @@ private PTableStats( throw new com.google.protobuf.InvalidProtocolBufferException( e.getMessage()).setUnfinishedMessage(this); } finally { - if (((mutable_bitField0_ & 0x00000002) == 0x00000002)) { - values_ = java.util.Collections.unmodifiableList(values_); - } this.unknownFields = unknownFields.build(); makeExtensionsImmutable(); } @@ -2052,27 +2057,20 @@ public com.google.protobuf.ByteString getKey() { return key_; } - // repeated bytes values = 2; - public static final int VALUES_FIELD_NUMBER = 2; - private java.util.List values_; + // optional bytes guidePosts = 2; + public static final int GUIDEPOSTS_FIELD_NUMBER = 2; + private com.google.protobuf.ByteString guidePosts_; /** - * repeated bytes values = 2; + * optional bytes guidePosts = 2; */ - public java.util.List - getValuesList() { - return values_; - } - /** - * repeated bytes values = 2; - */ - public int getValuesCount() { - return values_.size(); + public boolean hasGuidePosts() { + return ((bitField0_ & 0x00000002) == 0x00000002); } /** - * repeated bytes values = 2; + * optional bytes guidePosts = 2; */ - public com.google.protobuf.ByteString getValues(int index) { - return values_.get(index); + public com.google.protobuf.ByteString getGuidePosts() { + return guidePosts_; } // optional int64 guidePostsByteCount = 3; @@ -2082,7 +2080,7 @@ public com.google.protobuf.ByteString getValues(int index) { * optional int64 guidePostsByteCount = 3; */ public boolean hasGuidePostsByteCount() { - return ((bitField0_ & 0x00000002) == 0x00000002); + return ((bitField0_ & 0x00000004) == 0x00000004); } /** * optional int64 guidePostsByteCount = 3; @@ -2098,7 +2096,7 @@ public long getGuidePostsByteCount() { * optional int64 keyBytesCount = 4; */ public boolean hasKeyBytesCount() { - return ((bitField0_ & 0x00000004) == 0x00000004); + return ((bitField0_ & 0x00000008) == 0x00000008); } /** * optional int64 keyBytesCount = 4; @@ -2114,7 +2112,7 @@ public long getKeyBytesCount() { * optional int32 guidePostsCount = 5; */ public boolean hasGuidePostsCount() { - return ((bitField0_ & 0x00000008) == 0x00000008); + return ((bitField0_ & 0x00000010) == 0x00000010); } /** * optional int32 guidePostsCount = 5; @@ -2130,7 +2128,7 @@ public int getGuidePostsCount() { * optional .PGuidePosts pGuidePosts = 6; */ public boolean hasPGuidePosts() { - return ((bitField0_ & 0x00000010) == 0x00000010); + return ((bitField0_ & 0x00000020) == 0x00000020); } /** * optional .PGuidePosts pGuidePosts = 6; @@ -2145,13 +2143,30 @@ public org.apache.phoenix.coprocessor.generated.PGuidePostsProtos.PGuidePostsOrB return pGuidePosts_; } + // optional int32 maxLength = 7; + public static final int MAXLENGTH_FIELD_NUMBER = 7; + private int maxLength_; + /** + * optional int32 maxLength = 7; + */ + public boolean hasMaxLength() { + return ((bitField0_ & 0x00000040) == 0x00000040); + } + /** + * optional int32 maxLength = 7; + */ + public int getMaxLength() { + return maxLength_; + } + private void initFields() { key_ = com.google.protobuf.ByteString.EMPTY; - values_ = java.util.Collections.emptyList(); + guidePosts_ = com.google.protobuf.ByteString.EMPTY; guidePostsByteCount_ = 0L; keyBytesCount_ = 0L; guidePostsCount_ = 0; pGuidePosts_ = org.apache.phoenix.coprocessor.generated.PGuidePostsProtos.PGuidePosts.getDefaultInstance(); + maxLength_ = 0; } private byte memoizedIsInitialized = -1; public final boolean isInitialized() { @@ -2172,21 +2187,24 @@ public void writeTo(com.google.protobuf.CodedOutputStream output) if (((bitField0_ & 0x00000001) == 0x00000001)) { output.writeBytes(1, key_); } - for (int i = 0; i < values_.size(); i++) { - output.writeBytes(2, values_.get(i)); - } if (((bitField0_ & 0x00000002) == 0x00000002)) { - output.writeInt64(3, guidePostsByteCount_); + output.writeBytes(2, guidePosts_); } if (((bitField0_ & 0x00000004) == 0x00000004)) { - output.writeInt64(4, keyBytesCount_); + output.writeInt64(3, guidePostsByteCount_); } if (((bitField0_ & 0x00000008) == 0x00000008)) { - output.writeInt32(5, guidePostsCount_); + output.writeInt64(4, keyBytesCount_); } if (((bitField0_ & 0x00000010) == 0x00000010)) { + output.writeInt32(5, guidePostsCount_); + } + if (((bitField0_ & 0x00000020) == 0x00000020)) { output.writeMessage(6, pGuidePosts_); } + if (((bitField0_ & 0x00000040) == 0x00000040)) { + output.writeInt32(7, maxLength_); + } getUnknownFields().writeTo(output); } @@ -2200,31 +2218,30 @@ public int getSerializedSize() { size += com.google.protobuf.CodedOutputStream .computeBytesSize(1, key_); } - { - int dataSize = 0; - for (int i = 0; i < values_.size(); i++) { - dataSize += com.google.protobuf.CodedOutputStream - .computeBytesSizeNoTag(values_.get(i)); - } - size += dataSize; - size += 1 * getValuesList().size(); - } if (((bitField0_ & 0x00000002) == 0x00000002)) { size += com.google.protobuf.CodedOutputStream - .computeInt64Size(3, guidePostsByteCount_); + .computeBytesSize(2, guidePosts_); } if (((bitField0_ & 0x00000004) == 0x00000004)) { size += com.google.protobuf.CodedOutputStream - .computeInt64Size(4, keyBytesCount_); + .computeInt64Size(3, guidePostsByteCount_); } if (((bitField0_ & 0x00000008) == 0x00000008)) { size += com.google.protobuf.CodedOutputStream - .computeInt32Size(5, guidePostsCount_); + .computeInt64Size(4, keyBytesCount_); } if (((bitField0_ & 0x00000010) == 0x00000010)) { + size += com.google.protobuf.CodedOutputStream + .computeInt32Size(5, guidePostsCount_); + } + if (((bitField0_ & 0x00000020) == 0x00000020)) { size += com.google.protobuf.CodedOutputStream .computeMessageSize(6, pGuidePosts_); } + if (((bitField0_ & 0x00000040) == 0x00000040)) { + size += com.google.protobuf.CodedOutputStream + .computeInt32Size(7, maxLength_); + } size += getUnknownFields().getSerializedSize(); memoizedSerializedSize = size; return size; @@ -2253,8 +2270,11 @@ public boolean equals(final java.lang.Object obj) { result = result && getKey() .equals(other.getKey()); } - result = result && getValuesList() - .equals(other.getValuesList()); + result = result && (hasGuidePosts() == other.hasGuidePosts()); + if (hasGuidePosts()) { + result = result && getGuidePosts() + .equals(other.getGuidePosts()); + } result = result && (hasGuidePostsByteCount() == other.hasGuidePostsByteCount()); if (hasGuidePostsByteCount()) { result = result && (getGuidePostsByteCount() @@ -2275,6 +2295,11 @@ public boolean equals(final java.lang.Object obj) { result = result && getPGuidePosts() .equals(other.getPGuidePosts()); } + result = result && (hasMaxLength() == other.hasMaxLength()); + if (hasMaxLength()) { + result = result && (getMaxLength() + == other.getMaxLength()); + } result = result && getUnknownFields().equals(other.getUnknownFields()); return result; @@ -2292,9 +2317,9 @@ public int hashCode() { hash = (37 * hash) + KEY_FIELD_NUMBER; hash = (53 * hash) + getKey().hashCode(); } - if (getValuesCount() > 0) { - hash = (37 * hash) + VALUES_FIELD_NUMBER; - hash = (53 * hash) + getValuesList().hashCode(); + if (hasGuidePosts()) { + hash = (37 * hash) + GUIDEPOSTS_FIELD_NUMBER; + hash = (53 * hash) + getGuidePosts().hashCode(); } if (hasGuidePostsByteCount()) { hash = (37 * hash) + GUIDEPOSTSBYTECOUNT_FIELD_NUMBER; @@ -2312,6 +2337,10 @@ public int hashCode() { hash = (37 * hash) + PGUIDEPOSTS_FIELD_NUMBER; hash = (53 * hash) + getPGuidePosts().hashCode(); } + if (hasMaxLength()) { + hash = (37 * hash) + MAXLENGTH_FIELD_NUMBER; + hash = (53 * hash) + getMaxLength(); + } hash = (29 * hash) + getUnknownFields().hashCode(); memoizedHashCode = hash; return hash; @@ -2424,7 +2453,7 @@ public Builder clear() { super.clear(); key_ = com.google.protobuf.ByteString.EMPTY; bitField0_ = (bitField0_ & ~0x00000001); - values_ = java.util.Collections.emptyList(); + guidePosts_ = com.google.protobuf.ByteString.EMPTY; bitField0_ = (bitField0_ & ~0x00000002); guidePostsByteCount_ = 0L; bitField0_ = (bitField0_ & ~0x00000004); @@ -2438,6 +2467,8 @@ public Builder clear() { pGuidePostsBuilder_.clear(); } bitField0_ = (bitField0_ & ~0x00000020); + maxLength_ = 0; + bitField0_ = (bitField0_ & ~0x00000040); return this; } @@ -2470,31 +2501,34 @@ public org.apache.phoenix.coprocessor.generated.PTableProtos.PTableStats buildPa to_bitField0_ |= 0x00000001; } result.key_ = key_; - if (((bitField0_ & 0x00000002) == 0x00000002)) { - values_ = java.util.Collections.unmodifiableList(values_); - bitField0_ = (bitField0_ & ~0x00000002); + if (((from_bitField0_ & 0x00000002) == 0x00000002)) { + to_bitField0_ |= 0x00000002; } - result.values_ = values_; + result.guidePosts_ = guidePosts_; if (((from_bitField0_ & 0x00000004) == 0x00000004)) { - to_bitField0_ |= 0x00000002; + to_bitField0_ |= 0x00000004; } result.guidePostsByteCount_ = guidePostsByteCount_; if (((from_bitField0_ & 0x00000008) == 0x00000008)) { - to_bitField0_ |= 0x00000004; + to_bitField0_ |= 0x00000008; } result.keyBytesCount_ = keyBytesCount_; if (((from_bitField0_ & 0x00000010) == 0x00000010)) { - to_bitField0_ |= 0x00000008; + to_bitField0_ |= 0x00000010; } result.guidePostsCount_ = guidePostsCount_; if (((from_bitField0_ & 0x00000020) == 0x00000020)) { - to_bitField0_ |= 0x00000010; + to_bitField0_ |= 0x00000020; } if (pGuidePostsBuilder_ == null) { result.pGuidePosts_ = pGuidePosts_; } else { result.pGuidePosts_ = pGuidePostsBuilder_.build(); } + if (((from_bitField0_ & 0x00000040) == 0x00000040)) { + to_bitField0_ |= 0x00000040; + } + result.maxLength_ = maxLength_; result.bitField0_ = to_bitField0_; onBuilt(); return result; @@ -2514,15 +2548,8 @@ public Builder mergeFrom(org.apache.phoenix.coprocessor.generated.PTableProtos.P if (other.hasKey()) { setKey(other.getKey()); } - if (!other.values_.isEmpty()) { - if (values_.isEmpty()) { - values_ = other.values_; - bitField0_ = (bitField0_ & ~0x00000002); - } else { - ensureValuesIsMutable(); - values_.addAll(other.values_); - } - onChanged(); + if (other.hasGuidePosts()) { + setGuidePosts(other.getGuidePosts()); } if (other.hasGuidePostsByteCount()) { setGuidePostsByteCount(other.getGuidePostsByteCount()); @@ -2536,6 +2563,9 @@ public Builder mergeFrom(org.apache.phoenix.coprocessor.generated.PTableProtos.P if (other.hasPGuidePosts()) { mergePGuidePosts(other.getPGuidePosts()); } + if (other.hasMaxLength()) { + setMaxLength(other.getMaxLength()); + } this.mergeUnknownFields(other.getUnknownFields()); return this; } @@ -2603,74 +2633,38 @@ public Builder clearKey() { return this; } - // repeated bytes values = 2; - private java.util.List values_ = java.util.Collections.emptyList(); - private void ensureValuesIsMutable() { - if (!((bitField0_ & 0x00000002) == 0x00000002)) { - values_ = new java.util.ArrayList(values_); - bitField0_ |= 0x00000002; - } - } - /** - * repeated bytes values = 2; - */ - public java.util.List - getValuesList() { - return java.util.Collections.unmodifiableList(values_); - } + // optional bytes guidePosts = 2; + private com.google.protobuf.ByteString guidePosts_ = com.google.protobuf.ByteString.EMPTY; /** - * repeated bytes values = 2; + * optional bytes guidePosts = 2; */ - public int getValuesCount() { - return values_.size(); + public boolean hasGuidePosts() { + return ((bitField0_ & 0x00000002) == 0x00000002); } /** - * repeated bytes values = 2; + * optional bytes guidePosts = 2; */ - public com.google.protobuf.ByteString getValues(int index) { - return values_.get(index); + public com.google.protobuf.ByteString getGuidePosts() { + return guidePosts_; } /** - * repeated bytes values = 2; + * optional bytes guidePosts = 2; */ - public Builder setValues( - int index, com.google.protobuf.ByteString value) { + public Builder setGuidePosts(com.google.protobuf.ByteString value) { if (value == null) { throw new NullPointerException(); } - ensureValuesIsMutable(); - values_.set(index, value); - onChanged(); - return this; - } - /** - * repeated bytes values = 2; - */ - public Builder addValues(com.google.protobuf.ByteString value) { - if (value == null) { - throw new NullPointerException(); - } - ensureValuesIsMutable(); - values_.add(value); - onChanged(); - return this; - } - /** - * repeated bytes values = 2; - */ - public Builder addAllValues( - java.lang.Iterable values) { - ensureValuesIsMutable(); - super.addAll(values, values_); + bitField0_ |= 0x00000002; + guidePosts_ = value; onChanged(); return this; } /** - * repeated bytes values = 2; + * optional bytes guidePosts = 2; */ - public Builder clearValues() { - values_ = java.util.Collections.emptyList(); + public Builder clearGuidePosts() { bitField0_ = (bitField0_ & ~0x00000002); + guidePosts_ = getDefaultInstance().getGuidePosts(); onChanged(); return this; } @@ -2891,6 +2885,39 @@ public org.apache.phoenix.coprocessor.generated.PGuidePostsProtos.PGuidePostsOrB return pGuidePostsBuilder_; } + // optional int32 maxLength = 7; + private int maxLength_ ; + /** + * optional int32 maxLength = 7; + */ + public boolean hasMaxLength() { + return ((bitField0_ & 0x00000040) == 0x00000040); + } + /** + * optional int32 maxLength = 7; + */ + public int getMaxLength() { + return maxLength_; + } + /** + * optional int32 maxLength = 7; + */ + public Builder setMaxLength(int value) { + bitField0_ |= 0x00000040; + maxLength_ = value; + onChanged(); + return this; + } + /** + * optional int32 maxLength = 7; + */ + public Builder clearMaxLength() { + bitField0_ = (bitField0_ & ~0x00000040); + maxLength_ = 0; + onChanged(); + return this; + } + // @@protoc_insertion_point(builder_scope:PTableStats) } @@ -6916,6 +6943,7 @@ public Builder clearUpdateCacheFrequency() { "\006 \002(\010\022\020\n\010position\030\007 \002(\005\022\021\n\tsortOrder\030\010 \002" + "(\005\022\021\n\tarraySize\030\t \001(\005\022\024\n\014viewConstant\030\n " + "\001(\014\022\026\n\016viewReferenced\030\013 \001(\010\022\022\n\nexpressio" + +<<<<<<< 3520e12858223dfcad343e0a3a29c71fe4d074bc "n\030\014 \001(\t\022\026\n\016isRowTimestamp\030\r \001(\010\"\232\001\n\013PTab" + "leStats\022\013\n\003key\030\001 \002(\014\022\016\n\006values\030\002 \003(\014\022\033\n\023" + "guidePostsByteCount\030\003 \001(\003\022\025\n\rkeyBytesCou", @@ -6941,6 +6969,33 @@ public Builder clearUpdateCacheFrequency() { "e\022\n\n\006SYSTEM\020\000\022\010\n\004USER\020\001\022\010\n\004VIEW\020\002\022\t\n\005IND", "EX\020\003\022\010\n\004JOIN\020\004B@\n(org.apache.phoenix.cop" + "rocessor.generatedB\014PTableProtosH\001\210\001\001\240\001\001" +======= + "n\030\014 \001(\t\022\026\n\016isRowTimestamp\030\r \001(\010\"\261\001\n\013PTab" + + "leStats\022\013\n\003key\030\001 \002(\014\022\022\n\nguidePosts\030\002 \001(\014" + + "\022\033\n\023guidePostsByteCount\030\003 \001(\003\022\025\n\rkeyByte", + "sCount\030\004 \001(\003\022\027\n\017guidePostsCount\030\005 \001(\005\022!\n" + + "\013pGuidePosts\030\006 \001(\0132\014.PGuidePosts\022\021\n\tmaxL" + + "ength\030\007 \001(\005\"\206\005\n\006PTable\022\027\n\017schemaNameByte" + + "s\030\001 \002(\014\022\026\n\016tableNameBytes\030\002 \002(\014\022\036\n\ttable" + + "Type\030\003 \002(\0162\013.PTableType\022\022\n\nindexState\030\004 " + + "\001(\t\022\026\n\016sequenceNumber\030\005 \002(\003\022\021\n\ttimeStamp" + + "\030\006 \002(\003\022\023\n\013pkNameBytes\030\007 \001(\014\022\021\n\tbucketNum" + + "\030\010 \002(\005\022\031\n\007columns\030\t \003(\0132\010.PColumn\022\030\n\007ind" + + "exes\030\n \003(\0132\007.PTable\022\027\n\017isImmutableRows\030\013" + + " \002(\010\022 \n\nguidePosts\030\014 \003(\0132\014.PTableStats\022\032", + "\n\022dataTableNameBytes\030\r \001(\014\022\031\n\021defaultFam" + + "ilyName\030\016 \001(\014\022\022\n\ndisableWAL\030\017 \002(\010\022\023\n\013mul" + + "tiTenant\030\020 \002(\010\022\020\n\010viewType\030\021 \001(\014\022\025\n\rview" + + "Statement\030\022 \001(\014\022\025\n\rphysicalNames\030\023 \003(\014\022\020" + + "\n\010tenantId\030\024 \001(\014\022\023\n\013viewIndexId\030\025 \001(\005\022\021\n" + + "\tindexType\030\026 \001(\014\022\026\n\016statsTimeStamp\030\027 \001(\003" + + "\022\022\n\nstoreNulls\030\030 \001(\010\022\027\n\017baseColumnCount\030" + + "\031 \001(\005\022\036\n\026rowKeyOrderOptimizable\030\032 \001(\010\022\025\n" + + "\rtransactional\030\033 \001(\010*A\n\nPTableType\022\n\n\006SY" + + "STEM\020\000\022\010\n\004USER\020\001\022\010\n\004VIEW\020\002\022\t\n\005INDEX\020\003\022\010\n", + "\004JOIN\020\004B@\n(org.apache.phoenix.coprocesso" + + "r.generatedB\014PTableProtosH\001\210\001\001\240\001\001" +>>>>>>> PHOENIX-2417 Compress memory used by row key byte[] of guideposts }; com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner = new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() { @@ -6958,7 +7013,7 @@ public com.google.protobuf.ExtensionRegistry assignDescriptors( internal_static_PTableStats_fieldAccessorTable = new com.google.protobuf.GeneratedMessage.FieldAccessorTable( internal_static_PTableStats_descriptor, - new java.lang.String[] { "Key", "Values", "GuidePostsByteCount", "KeyBytesCount", "GuidePostsCount", "PGuidePosts", }); + new java.lang.String[] { "Key", "GuidePosts", "GuidePostsByteCount", "KeyBytesCount", "GuidePostsCount", "PGuidePosts", "MaxLength", }); internal_static_PTable_descriptor = getDescriptor().getMessageTypes().get(2); internal_static_PTable_fieldAccessorTable = new diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/StatCollectorProtos.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/StatCollectorProtos.java deleted file mode 100644 index 5008f22a303..00000000000 --- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/StatCollectorProtos.java +++ /dev/null @@ -1,1269 +0,0 @@ -// Generated by the protocol buffer compiler. DO NOT EDIT! -// source: StatisticsCollect.proto - -package org.apache.phoenix.coprocessor.generated; - -public final class StatCollectorProtos { - private StatCollectorProtos() {} - public static void registerAllExtensions( - com.google.protobuf.ExtensionRegistry registry) { - } - public interface StatCollectRequestOrBuilder - extends com.google.protobuf.MessageOrBuilder { - - // optional bytes startRow = 1; - /** - * optional bytes startRow = 1; - */ - boolean hasStartRow(); - /** - * optional bytes startRow = 1; - */ - com.google.protobuf.ByteString getStartRow(); - - // optional bytes stopRow = 2; - /** - * optional bytes stopRow = 2; - */ - boolean hasStopRow(); - /** - * optional bytes stopRow = 2; - */ - com.google.protobuf.ByteString getStopRow(); - } - /** - * Protobuf type {@code StatCollectRequest} - */ - public static final class StatCollectRequest extends - com.google.protobuf.GeneratedMessage - implements StatCollectRequestOrBuilder { - // Use StatCollectRequest.newBuilder() to construct. - private StatCollectRequest(com.google.protobuf.GeneratedMessage.Builder builder) { - super(builder); - this.unknownFields = builder.getUnknownFields(); - } - private StatCollectRequest(boolean noInit) { this.unknownFields = com.google.protobuf.UnknownFieldSet.getDefaultInstance(); } - - private static final StatCollectRequest defaultInstance; - public static StatCollectRequest getDefaultInstance() { - return defaultInstance; - } - - public StatCollectRequest getDefaultInstanceForType() { - return defaultInstance; - } - - private final com.google.protobuf.UnknownFieldSet unknownFields; - @java.lang.Override - public final com.google.protobuf.UnknownFieldSet - getUnknownFields() { - return this.unknownFields; - } - private StatCollectRequest( - com.google.protobuf.CodedInputStream input, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws com.google.protobuf.InvalidProtocolBufferException { - initFields(); - int mutable_bitField0_ = 0; - com.google.protobuf.UnknownFieldSet.Builder unknownFields = - com.google.protobuf.UnknownFieldSet.newBuilder(); - try { - boolean done = false; - while (!done) { - int tag = input.readTag(); - switch (tag) { - case 0: - done = true; - break; - default: { - if (!parseUnknownField(input, unknownFields, - extensionRegistry, tag)) { - done = true; - } - break; - } - case 10: { - bitField0_ |= 0x00000001; - startRow_ = input.readBytes(); - break; - } - case 18: { - bitField0_ |= 0x00000002; - stopRow_ = input.readBytes(); - break; - } - } - } - } catch (com.google.protobuf.InvalidProtocolBufferException e) { - throw e.setUnfinishedMessage(this); - } catch (java.io.IOException e) { - throw new com.google.protobuf.InvalidProtocolBufferException( - e.getMessage()).setUnfinishedMessage(this); - } finally { - this.unknownFields = unknownFields.build(); - makeExtensionsImmutable(); - } - } - public static final com.google.protobuf.Descriptors.Descriptor - getDescriptor() { - return org.apache.phoenix.coprocessor.generated.StatCollectorProtos.internal_static_StatCollectRequest_descriptor; - } - - protected com.google.protobuf.GeneratedMessage.FieldAccessorTable - internalGetFieldAccessorTable() { - return org.apache.phoenix.coprocessor.generated.StatCollectorProtos.internal_static_StatCollectRequest_fieldAccessorTable - .ensureFieldAccessorsInitialized( - org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectRequest.class, org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectRequest.Builder.class); - } - - public static com.google.protobuf.Parser PARSER = - new com.google.protobuf.AbstractParser() { - public StatCollectRequest parsePartialFrom( - com.google.protobuf.CodedInputStream input, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws com.google.protobuf.InvalidProtocolBufferException { - return new StatCollectRequest(input, extensionRegistry); - } - }; - - @java.lang.Override - public com.google.protobuf.Parser getParserForType() { - return PARSER; - } - - private int bitField0_; - // optional bytes startRow = 1; - public static final int STARTROW_FIELD_NUMBER = 1; - private com.google.protobuf.ByteString startRow_; - /** - * optional bytes startRow = 1; - */ - public boolean hasStartRow() { - return ((bitField0_ & 0x00000001) == 0x00000001); - } - /** - * optional bytes startRow = 1; - */ - public com.google.protobuf.ByteString getStartRow() { - return startRow_; - } - - // optional bytes stopRow = 2; - public static final int STOPROW_FIELD_NUMBER = 2; - private com.google.protobuf.ByteString stopRow_; - /** - * optional bytes stopRow = 2; - */ - public boolean hasStopRow() { - return ((bitField0_ & 0x00000002) == 0x00000002); - } - /** - * optional bytes stopRow = 2; - */ - public com.google.protobuf.ByteString getStopRow() { - return stopRow_; - } - - private void initFields() { - startRow_ = com.google.protobuf.ByteString.EMPTY; - stopRow_ = com.google.protobuf.ByteString.EMPTY; - } - private byte memoizedIsInitialized = -1; - public final boolean isInitialized() { - byte isInitialized = memoizedIsInitialized; - if (isInitialized != -1) return isInitialized == 1; - - memoizedIsInitialized = 1; - return true; - } - - public void writeTo(com.google.protobuf.CodedOutputStream output) - throws java.io.IOException { - getSerializedSize(); - if (((bitField0_ & 0x00000001) == 0x00000001)) { - output.writeBytes(1, startRow_); - } - if (((bitField0_ & 0x00000002) == 0x00000002)) { - output.writeBytes(2, stopRow_); - } - getUnknownFields().writeTo(output); - } - - private int memoizedSerializedSize = -1; - public int getSerializedSize() { - int size = memoizedSerializedSize; - if (size != -1) return size; - - size = 0; - if (((bitField0_ & 0x00000001) == 0x00000001)) { - size += com.google.protobuf.CodedOutputStream - .computeBytesSize(1, startRow_); - } - if (((bitField0_ & 0x00000002) == 0x00000002)) { - size += com.google.protobuf.CodedOutputStream - .computeBytesSize(2, stopRow_); - } - size += getUnknownFields().getSerializedSize(); - memoizedSerializedSize = size; - return size; - } - - private static final long serialVersionUID = 0L; - @java.lang.Override - protected java.lang.Object writeReplace() - throws java.io.ObjectStreamException { - return super.writeReplace(); - } - - @java.lang.Override - public boolean equals(final java.lang.Object obj) { - if (obj == this) { - return true; - } - if (!(obj instanceof org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectRequest)) { - return super.equals(obj); - } - org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectRequest other = (org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectRequest) obj; - - boolean result = true; - result = result && (hasStartRow() == other.hasStartRow()); - if (hasStartRow()) { - result = result && getStartRow() - .equals(other.getStartRow()); - } - result = result && (hasStopRow() == other.hasStopRow()); - if (hasStopRow()) { - result = result && getStopRow() - .equals(other.getStopRow()); - } - result = result && - getUnknownFields().equals(other.getUnknownFields()); - return result; - } - - private int memoizedHashCode = 0; - @java.lang.Override - public int hashCode() { - if (memoizedHashCode != 0) { - return memoizedHashCode; - } - int hash = 41; - hash = (19 * hash) + getDescriptorForType().hashCode(); - if (hasStartRow()) { - hash = (37 * hash) + STARTROW_FIELD_NUMBER; - hash = (53 * hash) + getStartRow().hashCode(); - } - if (hasStopRow()) { - hash = (37 * hash) + STOPROW_FIELD_NUMBER; - hash = (53 * hash) + getStopRow().hashCode(); - } - hash = (29 * hash) + getUnknownFields().hashCode(); - memoizedHashCode = hash; - return hash; - } - - public static org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectRequest parseFrom( - com.google.protobuf.ByteString data) - throws com.google.protobuf.InvalidProtocolBufferException { - return PARSER.parseFrom(data); - } - public static org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectRequest parseFrom( - com.google.protobuf.ByteString data, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws com.google.protobuf.InvalidProtocolBufferException { - return PARSER.parseFrom(data, extensionRegistry); - } - public static org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectRequest parseFrom(byte[] data) - throws com.google.protobuf.InvalidProtocolBufferException { - return PARSER.parseFrom(data); - } - public static org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectRequest parseFrom( - byte[] data, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws com.google.protobuf.InvalidProtocolBufferException { - return PARSER.parseFrom(data, extensionRegistry); - } - public static org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectRequest parseFrom(java.io.InputStream input) - throws java.io.IOException { - return PARSER.parseFrom(input); - } - public static org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectRequest parseFrom( - java.io.InputStream input, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws java.io.IOException { - return PARSER.parseFrom(input, extensionRegistry); - } - public static org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectRequest parseDelimitedFrom(java.io.InputStream input) - throws java.io.IOException { - return PARSER.parseDelimitedFrom(input); - } - public static org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectRequest parseDelimitedFrom( - java.io.InputStream input, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws java.io.IOException { - return PARSER.parseDelimitedFrom(input, extensionRegistry); - } - public static org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectRequest parseFrom( - com.google.protobuf.CodedInputStream input) - throws java.io.IOException { - return PARSER.parseFrom(input); - } - public static org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectRequest parseFrom( - com.google.protobuf.CodedInputStream input, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws java.io.IOException { - return PARSER.parseFrom(input, extensionRegistry); - } - - public static Builder newBuilder() { return Builder.create(); } - public Builder newBuilderForType() { return newBuilder(); } - public static Builder newBuilder(org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectRequest prototype) { - return newBuilder().mergeFrom(prototype); - } - public Builder toBuilder() { return newBuilder(this); } - - @java.lang.Override - protected Builder newBuilderForType( - com.google.protobuf.GeneratedMessage.BuilderParent parent) { - Builder builder = new Builder(parent); - return builder; - } - /** - * Protobuf type {@code StatCollectRequest} - */ - public static final class Builder extends - com.google.protobuf.GeneratedMessage.Builder - implements org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectRequestOrBuilder { - public static final com.google.protobuf.Descriptors.Descriptor - getDescriptor() { - return org.apache.phoenix.coprocessor.generated.StatCollectorProtos.internal_static_StatCollectRequest_descriptor; - } - - protected com.google.protobuf.GeneratedMessage.FieldAccessorTable - internalGetFieldAccessorTable() { - return org.apache.phoenix.coprocessor.generated.StatCollectorProtos.internal_static_StatCollectRequest_fieldAccessorTable - .ensureFieldAccessorsInitialized( - org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectRequest.class, org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectRequest.Builder.class); - } - - // Construct using org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectRequest.newBuilder() - private Builder() { - maybeForceBuilderInitialization(); - } - - private Builder( - com.google.protobuf.GeneratedMessage.BuilderParent parent) { - super(parent); - maybeForceBuilderInitialization(); - } - private void maybeForceBuilderInitialization() { - if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) { - } - } - private static Builder create() { - return new Builder(); - } - - public Builder clear() { - super.clear(); - startRow_ = com.google.protobuf.ByteString.EMPTY; - bitField0_ = (bitField0_ & ~0x00000001); - stopRow_ = com.google.protobuf.ByteString.EMPTY; - bitField0_ = (bitField0_ & ~0x00000002); - return this; - } - - public Builder clone() { - return create().mergeFrom(buildPartial()); - } - - public com.google.protobuf.Descriptors.Descriptor - getDescriptorForType() { - return org.apache.phoenix.coprocessor.generated.StatCollectorProtos.internal_static_StatCollectRequest_descriptor; - } - - public org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectRequest getDefaultInstanceForType() { - return org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectRequest.getDefaultInstance(); - } - - public org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectRequest build() { - org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectRequest result = buildPartial(); - if (!result.isInitialized()) { - throw newUninitializedMessageException(result); - } - return result; - } - - public org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectRequest buildPartial() { - org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectRequest result = new org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectRequest(this); - int from_bitField0_ = bitField0_; - int to_bitField0_ = 0; - if (((from_bitField0_ & 0x00000001) == 0x00000001)) { - to_bitField0_ |= 0x00000001; - } - result.startRow_ = startRow_; - if (((from_bitField0_ & 0x00000002) == 0x00000002)) { - to_bitField0_ |= 0x00000002; - } - result.stopRow_ = stopRow_; - result.bitField0_ = to_bitField0_; - onBuilt(); - return result; - } - - public Builder mergeFrom(com.google.protobuf.Message other) { - if (other instanceof org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectRequest) { - return mergeFrom((org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectRequest)other); - } else { - super.mergeFrom(other); - return this; - } - } - - public Builder mergeFrom(org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectRequest other) { - if (other == org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectRequest.getDefaultInstance()) return this; - if (other.hasStartRow()) { - setStartRow(other.getStartRow()); - } - if (other.hasStopRow()) { - setStopRow(other.getStopRow()); - } - this.mergeUnknownFields(other.getUnknownFields()); - return this; - } - - public final boolean isInitialized() { - return true; - } - - public Builder mergeFrom( - com.google.protobuf.CodedInputStream input, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws java.io.IOException { - org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectRequest parsedMessage = null; - try { - parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry); - } catch (com.google.protobuf.InvalidProtocolBufferException e) { - parsedMessage = (org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectRequest) e.getUnfinishedMessage(); - throw e; - } finally { - if (parsedMessage != null) { - mergeFrom(parsedMessage); - } - } - return this; - } - private int bitField0_; - - // optional bytes startRow = 1; - private com.google.protobuf.ByteString startRow_ = com.google.protobuf.ByteString.EMPTY; - /** - * optional bytes startRow = 1; - */ - public boolean hasStartRow() { - return ((bitField0_ & 0x00000001) == 0x00000001); - } - /** - * optional bytes startRow = 1; - */ - public com.google.protobuf.ByteString getStartRow() { - return startRow_; - } - /** - * optional bytes startRow = 1; - */ - public Builder setStartRow(com.google.protobuf.ByteString value) { - if (value == null) { - throw new NullPointerException(); - } - bitField0_ |= 0x00000001; - startRow_ = value; - onChanged(); - return this; - } - /** - * optional bytes startRow = 1; - */ - public Builder clearStartRow() { - bitField0_ = (bitField0_ & ~0x00000001); - startRow_ = getDefaultInstance().getStartRow(); - onChanged(); - return this; - } - - // optional bytes stopRow = 2; - private com.google.protobuf.ByteString stopRow_ = com.google.protobuf.ByteString.EMPTY; - /** - * optional bytes stopRow = 2; - */ - public boolean hasStopRow() { - return ((bitField0_ & 0x00000002) == 0x00000002); - } - /** - * optional bytes stopRow = 2; - */ - public com.google.protobuf.ByteString getStopRow() { - return stopRow_; - } - /** - * optional bytes stopRow = 2; - */ - public Builder setStopRow(com.google.protobuf.ByteString value) { - if (value == null) { - throw new NullPointerException(); - } - bitField0_ |= 0x00000002; - stopRow_ = value; - onChanged(); - return this; - } - /** - * optional bytes stopRow = 2; - */ - public Builder clearStopRow() { - bitField0_ = (bitField0_ & ~0x00000002); - stopRow_ = getDefaultInstance().getStopRow(); - onChanged(); - return this; - } - - // @@protoc_insertion_point(builder_scope:StatCollectRequest) - } - - static { - defaultInstance = new StatCollectRequest(true); - defaultInstance.initFields(); - } - - // @@protoc_insertion_point(class_scope:StatCollectRequest) - } - - public interface StatCollectResponseOrBuilder - extends com.google.protobuf.MessageOrBuilder { - - // required uint64 rowsScanned = 1; - /** - * required uint64 rowsScanned = 1; - */ - boolean hasRowsScanned(); - /** - * required uint64 rowsScanned = 1; - */ - long getRowsScanned(); - } - /** - * Protobuf type {@code StatCollectResponse} - */ - public static final class StatCollectResponse extends - com.google.protobuf.GeneratedMessage - implements StatCollectResponseOrBuilder { - // Use StatCollectResponse.newBuilder() to construct. - private StatCollectResponse(com.google.protobuf.GeneratedMessage.Builder builder) { - super(builder); - this.unknownFields = builder.getUnknownFields(); - } - private StatCollectResponse(boolean noInit) { this.unknownFields = com.google.protobuf.UnknownFieldSet.getDefaultInstance(); } - - private static final StatCollectResponse defaultInstance; - public static StatCollectResponse getDefaultInstance() { - return defaultInstance; - } - - public StatCollectResponse getDefaultInstanceForType() { - return defaultInstance; - } - - private final com.google.protobuf.UnknownFieldSet unknownFields; - @java.lang.Override - public final com.google.protobuf.UnknownFieldSet - getUnknownFields() { - return this.unknownFields; - } - private StatCollectResponse( - com.google.protobuf.CodedInputStream input, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws com.google.protobuf.InvalidProtocolBufferException { - initFields(); - int mutable_bitField0_ = 0; - com.google.protobuf.UnknownFieldSet.Builder unknownFields = - com.google.protobuf.UnknownFieldSet.newBuilder(); - try { - boolean done = false; - while (!done) { - int tag = input.readTag(); - switch (tag) { - case 0: - done = true; - break; - default: { - if (!parseUnknownField(input, unknownFields, - extensionRegistry, tag)) { - done = true; - } - break; - } - case 8: { - bitField0_ |= 0x00000001; - rowsScanned_ = input.readUInt64(); - break; - } - } - } - } catch (com.google.protobuf.InvalidProtocolBufferException e) { - throw e.setUnfinishedMessage(this); - } catch (java.io.IOException e) { - throw new com.google.protobuf.InvalidProtocolBufferException( - e.getMessage()).setUnfinishedMessage(this); - } finally { - this.unknownFields = unknownFields.build(); - makeExtensionsImmutable(); - } - } - public static final com.google.protobuf.Descriptors.Descriptor - getDescriptor() { - return org.apache.phoenix.coprocessor.generated.StatCollectorProtos.internal_static_StatCollectResponse_descriptor; - } - - protected com.google.protobuf.GeneratedMessage.FieldAccessorTable - internalGetFieldAccessorTable() { - return org.apache.phoenix.coprocessor.generated.StatCollectorProtos.internal_static_StatCollectResponse_fieldAccessorTable - .ensureFieldAccessorsInitialized( - org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectResponse.class, org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectResponse.Builder.class); - } - - public static com.google.protobuf.Parser PARSER = - new com.google.protobuf.AbstractParser() { - public StatCollectResponse parsePartialFrom( - com.google.protobuf.CodedInputStream input, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws com.google.protobuf.InvalidProtocolBufferException { - return new StatCollectResponse(input, extensionRegistry); - } - }; - - @java.lang.Override - public com.google.protobuf.Parser getParserForType() { - return PARSER; - } - - private int bitField0_; - // required uint64 rowsScanned = 1; - public static final int ROWSSCANNED_FIELD_NUMBER = 1; - private long rowsScanned_; - /** - * required uint64 rowsScanned = 1; - */ - public boolean hasRowsScanned() { - return ((bitField0_ & 0x00000001) == 0x00000001); - } - /** - * required uint64 rowsScanned = 1; - */ - public long getRowsScanned() { - return rowsScanned_; - } - - private void initFields() { - rowsScanned_ = 0L; - } - private byte memoizedIsInitialized = -1; - public final boolean isInitialized() { - byte isInitialized = memoizedIsInitialized; - if (isInitialized != -1) return isInitialized == 1; - - if (!hasRowsScanned()) { - memoizedIsInitialized = 0; - return false; - } - memoizedIsInitialized = 1; - return true; - } - - public void writeTo(com.google.protobuf.CodedOutputStream output) - throws java.io.IOException { - getSerializedSize(); - if (((bitField0_ & 0x00000001) == 0x00000001)) { - output.writeUInt64(1, rowsScanned_); - } - getUnknownFields().writeTo(output); - } - - private int memoizedSerializedSize = -1; - public int getSerializedSize() { - int size = memoizedSerializedSize; - if (size != -1) return size; - - size = 0; - if (((bitField0_ & 0x00000001) == 0x00000001)) { - size += com.google.protobuf.CodedOutputStream - .computeUInt64Size(1, rowsScanned_); - } - size += getUnknownFields().getSerializedSize(); - memoizedSerializedSize = size; - return size; - } - - private static final long serialVersionUID = 0L; - @java.lang.Override - protected java.lang.Object writeReplace() - throws java.io.ObjectStreamException { - return super.writeReplace(); - } - - @java.lang.Override - public boolean equals(final java.lang.Object obj) { - if (obj == this) { - return true; - } - if (!(obj instanceof org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectResponse)) { - return super.equals(obj); - } - org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectResponse other = (org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectResponse) obj; - - boolean result = true; - result = result && (hasRowsScanned() == other.hasRowsScanned()); - if (hasRowsScanned()) { - result = result && (getRowsScanned() - == other.getRowsScanned()); - } - result = result && - getUnknownFields().equals(other.getUnknownFields()); - return result; - } - - private int memoizedHashCode = 0; - @java.lang.Override - public int hashCode() { - if (memoizedHashCode != 0) { - return memoizedHashCode; - } - int hash = 41; - hash = (19 * hash) + getDescriptorForType().hashCode(); - if (hasRowsScanned()) { - hash = (37 * hash) + ROWSSCANNED_FIELD_NUMBER; - hash = (53 * hash) + hashLong(getRowsScanned()); - } - hash = (29 * hash) + getUnknownFields().hashCode(); - memoizedHashCode = hash; - return hash; - } - - public static org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectResponse parseFrom( - com.google.protobuf.ByteString data) - throws com.google.protobuf.InvalidProtocolBufferException { - return PARSER.parseFrom(data); - } - public static org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectResponse parseFrom( - com.google.protobuf.ByteString data, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws com.google.protobuf.InvalidProtocolBufferException { - return PARSER.parseFrom(data, extensionRegistry); - } - public static org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectResponse parseFrom(byte[] data) - throws com.google.protobuf.InvalidProtocolBufferException { - return PARSER.parseFrom(data); - } - public static org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectResponse parseFrom( - byte[] data, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws com.google.protobuf.InvalidProtocolBufferException { - return PARSER.parseFrom(data, extensionRegistry); - } - public static org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectResponse parseFrom(java.io.InputStream input) - throws java.io.IOException { - return PARSER.parseFrom(input); - } - public static org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectResponse parseFrom( - java.io.InputStream input, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws java.io.IOException { - return PARSER.parseFrom(input, extensionRegistry); - } - public static org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectResponse parseDelimitedFrom(java.io.InputStream input) - throws java.io.IOException { - return PARSER.parseDelimitedFrom(input); - } - public static org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectResponse parseDelimitedFrom( - java.io.InputStream input, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws java.io.IOException { - return PARSER.parseDelimitedFrom(input, extensionRegistry); - } - public static org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectResponse parseFrom( - com.google.protobuf.CodedInputStream input) - throws java.io.IOException { - return PARSER.parseFrom(input); - } - public static org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectResponse parseFrom( - com.google.protobuf.CodedInputStream input, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws java.io.IOException { - return PARSER.parseFrom(input, extensionRegistry); - } - - public static Builder newBuilder() { return Builder.create(); } - public Builder newBuilderForType() { return newBuilder(); } - public static Builder newBuilder(org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectResponse prototype) { - return newBuilder().mergeFrom(prototype); - } - public Builder toBuilder() { return newBuilder(this); } - - @java.lang.Override - protected Builder newBuilderForType( - com.google.protobuf.GeneratedMessage.BuilderParent parent) { - Builder builder = new Builder(parent); - return builder; - } - /** - * Protobuf type {@code StatCollectResponse} - */ - public static final class Builder extends - com.google.protobuf.GeneratedMessage.Builder - implements org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectResponseOrBuilder { - public static final com.google.protobuf.Descriptors.Descriptor - getDescriptor() { - return org.apache.phoenix.coprocessor.generated.StatCollectorProtos.internal_static_StatCollectResponse_descriptor; - } - - protected com.google.protobuf.GeneratedMessage.FieldAccessorTable - internalGetFieldAccessorTable() { - return org.apache.phoenix.coprocessor.generated.StatCollectorProtos.internal_static_StatCollectResponse_fieldAccessorTable - .ensureFieldAccessorsInitialized( - org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectResponse.class, org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectResponse.Builder.class); - } - - // Construct using org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectResponse.newBuilder() - private Builder() { - maybeForceBuilderInitialization(); - } - - private Builder( - com.google.protobuf.GeneratedMessage.BuilderParent parent) { - super(parent); - maybeForceBuilderInitialization(); - } - private void maybeForceBuilderInitialization() { - if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) { - } - } - private static Builder create() { - return new Builder(); - } - - public Builder clear() { - super.clear(); - rowsScanned_ = 0L; - bitField0_ = (bitField0_ & ~0x00000001); - return this; - } - - public Builder clone() { - return create().mergeFrom(buildPartial()); - } - - public com.google.protobuf.Descriptors.Descriptor - getDescriptorForType() { - return org.apache.phoenix.coprocessor.generated.StatCollectorProtos.internal_static_StatCollectResponse_descriptor; - } - - public org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectResponse getDefaultInstanceForType() { - return org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectResponse.getDefaultInstance(); - } - - public org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectResponse build() { - org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectResponse result = buildPartial(); - if (!result.isInitialized()) { - throw newUninitializedMessageException(result); - } - return result; - } - - public org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectResponse buildPartial() { - org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectResponse result = new org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectResponse(this); - int from_bitField0_ = bitField0_; - int to_bitField0_ = 0; - if (((from_bitField0_ & 0x00000001) == 0x00000001)) { - to_bitField0_ |= 0x00000001; - } - result.rowsScanned_ = rowsScanned_; - result.bitField0_ = to_bitField0_; - onBuilt(); - return result; - } - - public Builder mergeFrom(com.google.protobuf.Message other) { - if (other instanceof org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectResponse) { - return mergeFrom((org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectResponse)other); - } else { - super.mergeFrom(other); - return this; - } - } - - public Builder mergeFrom(org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectResponse other) { - if (other == org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectResponse.getDefaultInstance()) return this; - if (other.hasRowsScanned()) { - setRowsScanned(other.getRowsScanned()); - } - this.mergeUnknownFields(other.getUnknownFields()); - return this; - } - - public final boolean isInitialized() { - if (!hasRowsScanned()) { - - return false; - } - return true; - } - - public Builder mergeFrom( - com.google.protobuf.CodedInputStream input, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws java.io.IOException { - org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectResponse parsedMessage = null; - try { - parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry); - } catch (com.google.protobuf.InvalidProtocolBufferException e) { - parsedMessage = (org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectResponse) e.getUnfinishedMessage(); - throw e; - } finally { - if (parsedMessage != null) { - mergeFrom(parsedMessage); - } - } - return this; - } - private int bitField0_; - - // required uint64 rowsScanned = 1; - private long rowsScanned_ ; - /** - * required uint64 rowsScanned = 1; - */ - public boolean hasRowsScanned() { - return ((bitField0_ & 0x00000001) == 0x00000001); - } - /** - * required uint64 rowsScanned = 1; - */ - public long getRowsScanned() { - return rowsScanned_; - } - /** - * required uint64 rowsScanned = 1; - */ - public Builder setRowsScanned(long value) { - bitField0_ |= 0x00000001; - rowsScanned_ = value; - onChanged(); - return this; - } - /** - * required uint64 rowsScanned = 1; - */ - public Builder clearRowsScanned() { - bitField0_ = (bitField0_ & ~0x00000001); - rowsScanned_ = 0L; - onChanged(); - return this; - } - - // @@protoc_insertion_point(builder_scope:StatCollectResponse) - } - - static { - defaultInstance = new StatCollectResponse(true); - defaultInstance.initFields(); - } - - // @@protoc_insertion_point(class_scope:StatCollectResponse) - } - - /** - * Protobuf service {@code StatCollectService} - */ - public static abstract class StatCollectService - implements com.google.protobuf.Service { - protected StatCollectService() {} - - public interface Interface { - /** - * rpc collectStat(.StatCollectRequest) returns (.StatCollectResponse); - */ - public abstract void collectStat( - com.google.protobuf.RpcController controller, - org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectRequest request, - com.google.protobuf.RpcCallback done); - - } - - public static com.google.protobuf.Service newReflectiveService( - final Interface impl) { - return new StatCollectService() { - @java.lang.Override - public void collectStat( - com.google.protobuf.RpcController controller, - org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectRequest request, - com.google.protobuf.RpcCallback done) { - impl.collectStat(controller, request, done); - } - - }; - } - - public static com.google.protobuf.BlockingService - newReflectiveBlockingService(final BlockingInterface impl) { - return new com.google.protobuf.BlockingService() { - public final com.google.protobuf.Descriptors.ServiceDescriptor - getDescriptorForType() { - return getDescriptor(); - } - - public final com.google.protobuf.Message callBlockingMethod( - com.google.protobuf.Descriptors.MethodDescriptor method, - com.google.protobuf.RpcController controller, - com.google.protobuf.Message request) - throws com.google.protobuf.ServiceException { - if (method.getService() != getDescriptor()) { - throw new java.lang.IllegalArgumentException( - "Service.callBlockingMethod() given method descriptor for " + - "wrong service type."); - } - switch(method.getIndex()) { - case 0: - return impl.collectStat(controller, (org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectRequest)request); - default: - throw new java.lang.AssertionError("Can't get here."); - } - } - - public final com.google.protobuf.Message - getRequestPrototype( - com.google.protobuf.Descriptors.MethodDescriptor method) { - if (method.getService() != getDescriptor()) { - throw new java.lang.IllegalArgumentException( - "Service.getRequestPrototype() given method " + - "descriptor for wrong service type."); - } - switch(method.getIndex()) { - case 0: - return org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectRequest.getDefaultInstance(); - default: - throw new java.lang.AssertionError("Can't get here."); - } - } - - public final com.google.protobuf.Message - getResponsePrototype( - com.google.protobuf.Descriptors.MethodDescriptor method) { - if (method.getService() != getDescriptor()) { - throw new java.lang.IllegalArgumentException( - "Service.getResponsePrototype() given method " + - "descriptor for wrong service type."); - } - switch(method.getIndex()) { - case 0: - return org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectResponse.getDefaultInstance(); - default: - throw new java.lang.AssertionError("Can't get here."); - } - } - - }; - } - - /** - * rpc collectStat(.StatCollectRequest) returns (.StatCollectResponse); - */ - public abstract void collectStat( - com.google.protobuf.RpcController controller, - org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectRequest request, - com.google.protobuf.RpcCallback done); - - public static final - com.google.protobuf.Descriptors.ServiceDescriptor - getDescriptor() { - return org.apache.phoenix.coprocessor.generated.StatCollectorProtos.getDescriptor().getServices().get(0); - } - public final com.google.protobuf.Descriptors.ServiceDescriptor - getDescriptorForType() { - return getDescriptor(); - } - - public final void callMethod( - com.google.protobuf.Descriptors.MethodDescriptor method, - com.google.protobuf.RpcController controller, - com.google.protobuf.Message request, - com.google.protobuf.RpcCallback< - com.google.protobuf.Message> done) { - if (method.getService() != getDescriptor()) { - throw new java.lang.IllegalArgumentException( - "Service.callMethod() given method descriptor for wrong " + - "service type."); - } - switch(method.getIndex()) { - case 0: - this.collectStat(controller, (org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectRequest)request, - com.google.protobuf.RpcUtil.specializeCallback( - done)); - return; - default: - throw new java.lang.AssertionError("Can't get here."); - } - } - - public final com.google.protobuf.Message - getRequestPrototype( - com.google.protobuf.Descriptors.MethodDescriptor method) { - if (method.getService() != getDescriptor()) { - throw new java.lang.IllegalArgumentException( - "Service.getRequestPrototype() given method " + - "descriptor for wrong service type."); - } - switch(method.getIndex()) { - case 0: - return org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectRequest.getDefaultInstance(); - default: - throw new java.lang.AssertionError("Can't get here."); - } - } - - public final com.google.protobuf.Message - getResponsePrototype( - com.google.protobuf.Descriptors.MethodDescriptor method) { - if (method.getService() != getDescriptor()) { - throw new java.lang.IllegalArgumentException( - "Service.getResponsePrototype() given method " + - "descriptor for wrong service type."); - } - switch(method.getIndex()) { - case 0: - return org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectResponse.getDefaultInstance(); - default: - throw new java.lang.AssertionError("Can't get here."); - } - } - - public static Stub newStub( - com.google.protobuf.RpcChannel channel) { - return new Stub(channel); - } - - public static final class Stub extends org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectService implements Interface { - private Stub(com.google.protobuf.RpcChannel channel) { - this.channel = channel; - } - - private final com.google.protobuf.RpcChannel channel; - - public com.google.protobuf.RpcChannel getChannel() { - return channel; - } - - public void collectStat( - com.google.protobuf.RpcController controller, - org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectRequest request, - com.google.protobuf.RpcCallback done) { - channel.callMethod( - getDescriptor().getMethods().get(0), - controller, - request, - org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectResponse.getDefaultInstance(), - com.google.protobuf.RpcUtil.generalizeCallback( - done, - org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectResponse.class, - org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectResponse.getDefaultInstance())); - } - } - - public static BlockingInterface newBlockingStub( - com.google.protobuf.BlockingRpcChannel channel) { - return new BlockingStub(channel); - } - - public interface BlockingInterface { - public org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectResponse collectStat( - com.google.protobuf.RpcController controller, - org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectRequest request) - throws com.google.protobuf.ServiceException; - } - - private static final class BlockingStub implements BlockingInterface { - private BlockingStub(com.google.protobuf.BlockingRpcChannel channel) { - this.channel = channel; - } - - private final com.google.protobuf.BlockingRpcChannel channel; - - public org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectResponse collectStat( - com.google.protobuf.RpcController controller, - org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectRequest request) - throws com.google.protobuf.ServiceException { - return (org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectResponse) channel.callBlockingMethod( - getDescriptor().getMethods().get(0), - controller, - request, - org.apache.phoenix.coprocessor.generated.StatCollectorProtos.StatCollectResponse.getDefaultInstance()); - } - - } - - // @@protoc_insertion_point(class_scope:StatCollectService) - } - - private static com.google.protobuf.Descriptors.Descriptor - internal_static_StatCollectRequest_descriptor; - private static - com.google.protobuf.GeneratedMessage.FieldAccessorTable - internal_static_StatCollectRequest_fieldAccessorTable; - private static com.google.protobuf.Descriptors.Descriptor - internal_static_StatCollectResponse_descriptor; - private static - com.google.protobuf.GeneratedMessage.FieldAccessorTable - internal_static_StatCollectResponse_fieldAccessorTable; - - public static com.google.protobuf.Descriptors.FileDescriptor - getDescriptor() { - return descriptor; - } - private static com.google.protobuf.Descriptors.FileDescriptor - descriptor; - static { - java.lang.String[] descriptorData = { - "\n\027StatisticsCollect.proto\"7\n\022StatCollect" + - "Request\022\020\n\010startRow\030\001 \001(\014\022\017\n\007stopRow\030\002 \001" + - "(\014\"*\n\023StatCollectResponse\022\023\n\013rowsScanned" + - "\030\001 \002(\0042N\n\022StatCollectService\0228\n\013collectS" + - "tat\022\023.StatCollectRequest\032\024.StatCollectRe" + - "sponseBG\n(org.apache.phoenix.coprocessor" + - ".generatedB\023StatCollectorProtosH\001\210\001\001\240\001\001" - }; - com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner = - new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() { - public com.google.protobuf.ExtensionRegistry assignDescriptors( - com.google.protobuf.Descriptors.FileDescriptor root) { - descriptor = root; - internal_static_StatCollectRequest_descriptor = - getDescriptor().getMessageTypes().get(0); - internal_static_StatCollectRequest_fieldAccessorTable = new - com.google.protobuf.GeneratedMessage.FieldAccessorTable( - internal_static_StatCollectRequest_descriptor, - new java.lang.String[] { "StartRow", "StopRow", }); - internal_static_StatCollectResponse_descriptor = - getDescriptor().getMessageTypes().get(1); - internal_static_StatCollectResponse_fieldAccessorTable = new - com.google.protobuf.GeneratedMessage.FieldAccessorTable( - internal_static_StatCollectResponse_descriptor, - new java.lang.String[] { "RowsScanned", }); - return null; - } - }; - com.google.protobuf.Descriptors.FileDescriptor - .internalBuildGeneratedFileFrom(descriptorData, - new com.google.protobuf.Descriptors.FileDescriptor[] { - }, assigner); - } - - // @@protoc_insertion_point(outer_class_scope) -} diff --git a/phoenix-core/src/main/java/org/apache/phoenix/execute/ScanPlan.java b/phoenix-core/src/main/java/org/apache/phoenix/execute/ScanPlan.java index 816cd1ba24b..efb00a8fff6 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/execute/ScanPlan.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/execute/ScanPlan.java @@ -122,7 +122,7 @@ private static boolean isSerial(StatementContext context, } else { // Region size estimated based on total number of bytes divided by number of regions long totByteSize = gpsInfo.getByteCount(); - estRegionSize = totByteSize / (gpsInfo.getGuidePosts().size()+1); + estRegionSize = totByteSize / (gpsInfo.getGuidePostsCount()+1); } // TODO: configurable number of bytes? boolean isSerial = (perScanLimit * estRowSize < estRegionSize); diff --git a/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java b/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java index 519b38f171c..1a5cdc55cfb 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java @@ -22,6 +22,11 @@ import static org.apache.phoenix.monitoring.GlobalClientMetrics.GLOBAL_QUERY_TIMEOUT_COUNTER; import static org.apache.phoenix.util.ByteUtil.EMPTY_BYTE_ARRAY; +import java.io.ByteArrayInputStream; +import java.io.DataInput; +import java.io.DataInputStream; +import java.io.EOFException; +import java.io.IOException; import java.sql.SQLException; import java.util.ArrayList; import java.util.Collections; @@ -46,6 +51,7 @@ import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.filter.FirstKeyOnlyFilter; import org.apache.hadoop.hbase.filter.PageFilter; +import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Pair; import org.apache.phoenix.compile.QueryPlan; @@ -75,7 +81,10 @@ import org.apache.phoenix.schema.TableRef; import org.apache.phoenix.schema.stats.GuidePostsInfo; import org.apache.phoenix.schema.stats.PTableStats; +import org.apache.phoenix.util.ByteUtil; +import org.apache.phoenix.util.CodecUtils; import org.apache.phoenix.util.LogUtil; +import org.apache.phoenix.util.PrefixByteDecoder; import org.apache.phoenix.util.SQLCloseables; import org.apache.phoenix.util.ScanUtil; import org.apache.phoenix.util.SchemaUtil; @@ -356,22 +365,18 @@ private static int getIndexContainingExclusive(List boundaries, byte[] e guideIndex = (guideIndex < 0 ? -(guideIndex + 1) : guideIndex); return guideIndex; } - - private List getGuidePosts(Set whereConditions) { + + private GuidePostsInfo getGuidePosts(Set whereConditions) { /* - * Don't use guide posts if: - * 1) We're doing a point lookup, as HBase is fast enough at those - * to not need them to be further parallelized. TODO: pref test to verify - * 2) We're collecting stats, as in this case we need to scan entire - * regions worth of data to track where to put the guide posts. + * Don't use guide posts if: 1) We're doing a point lookup, as HBase is fast enough at those to not need them to + * be further parallelized. TODO: pref test to verify 2) We're collecting stats, as in this case we need to scan + * entire regions worth of data to track where to put the guide posts. */ - if (!useStats()) { - return Collections.emptyList(); - } - - List gps = null; + if (!useStats()) { return GuidePostsInfo.EMPTY_GUIDEPOST; } + + GuidePostsInfo gps = null; PTable table = getTable(); - Map guidePostMap = tableStats.getGuidePosts(); + Map guidePostMap = tableStats.getGuidePosts(); byte[] defaultCF = SchemaUtil.getEmptyColumnFamily(getTable()); if (table.getColumnFamilies().isEmpty()) { // For sure we can get the defaultCF from the table @@ -379,37 +384,35 @@ private List getGuidePosts(Set whereConditions) { } else { byte[] familyInWhere = null; if (!whereConditions.isEmpty()) { - if (whereConditions.contains(defaultCF)) { - gps = getDefaultFamilyGuidePosts(guidePostMap, defaultCF); - } else { - familyInWhere = whereConditions.iterator().next(); - if(familyInWhere != null) { - GuidePostsInfo guidePostsInfo = guidePostMap.get(familyInWhere); - if (guidePostsInfo != null) { - gps = guidePostsInfo.getGuidePosts(); - } else { - // As there are no guideposts collected for the where family we go with the default CF + if (whereConditions.contains(defaultCF)) { gps = getDefaultFamilyGuidePosts(guidePostMap, defaultCF); - } + } else { + familyInWhere = whereConditions.iterator().next(); + if (familyInWhere != null) { + GuidePostsInfo guidePostsInfo = guidePostMap.get(familyInWhere); + if (guidePostsInfo != null) { + gps = guidePostsInfo; + } else { + // As there are no guideposts collected for the where family we go with the default CF + gps = getDefaultFamilyGuidePosts(guidePostMap, defaultCF); + } + } } - } } else { - gps = getDefaultFamilyGuidePosts(guidePostMap, defaultCF); + gps = getDefaultFamilyGuidePosts(guidePostMap, defaultCF); } } - if (gps == null) { - return Collections.emptyList(); - } + if (gps == null) { return GuidePostsInfo.EMPTY_GUIDEPOST; } return gps; } - private List getDefaultFamilyGuidePosts(Map guidePostMap, byte[] defaultCF) { - if (guidePostMap.get(defaultCF) != null) { - return guidePostMap.get(defaultCF).getGuidePosts(); - } - return null; + private GuidePostsInfo getDefaultFamilyGuidePosts(Map guidePostMap, byte[] defaultCF) { + if (guidePostMap.get(defaultCF) != null) { + return guidePostMap.get(defaultCF); + } + return null; } - + private static String toString(List gps) { StringBuilder buf = new StringBuilder(gps.size() * 100); buf.append("["); @@ -461,9 +464,9 @@ private List> getParallelScans(byte[] startKey, byte[] stopKey) throw for(Pair where : context.getWhereConditionColumns()) { whereConditions.add(where.getFirst()); } - List gps = getGuidePosts(whereConditions); + GuidePostsInfo gps = getGuidePosts(whereConditions); if (logger.isDebugEnabled()) { - logger.debug("Guideposts: " + toString(gps)); + logger.debug("Guideposts: " + gps); } boolean traverseAllRegions = isSalted || isLocalIndex; if (!traverseAllRegions) { @@ -490,15 +493,37 @@ private List> getParallelScans(byte[] startKey, byte[] stopKey) throw } List> parallelScans = Lists.newArrayListWithExpectedSize(stopIndex - regionIndex + 1); - byte[] currentKey = startKey; - int guideIndex = currentKey.length == 0 ? 0 : getIndexContainingInclusive(gps, currentKey); - int gpsSize = gps.size(); + ImmutableBytesWritable currentKey = new ImmutableBytesWritable(startKey, 0, startKey.length); + + int gpsSize = gps.getGuidePostsCount(); int estGuidepostsPerRegion = gpsSize == 0 ? 1 : gpsSize / regionLocations.size() + 1; int keyOffset = 0; + ImmutableBytesWritable currentGuidePost = ByteUtil.EMPTY_IMMUTABLE_BYTE_ARRAY; List scans = Lists.newArrayListWithExpectedSize(estGuidepostsPerRegion); + ImmutableBytesWritable guidePosts = gps.getGuidePosts(); + ByteArrayInputStream stream = null; + DataInput input = null; + PrefixByteDecoder decoder = null; + int guideIndex = 0; + if (gpsSize > 0) { + stream = new ByteArrayInputStream(guidePosts.get(), guidePosts.getOffset(), guidePosts.getLength()); + input = new DataInputStream(stream); + if (gps.getMaxLength() > 0) { + decoder = new PrefixByteDecoder(gps.getMaxLength()); + } else { + decoder = new PrefixByteDecoder(); + } + try { + currentGuidePost = CodecUtils.decode(decoder, input); + while (currentKey.getLength() != 0 && Bytes.compareTo(currentKey.get(), currentGuidePost.get()) <= 0) { + currentGuidePost = CodecUtils.decode(decoder, input); + guideIndex++; + } + } catch (EOFException e) {} + } // Merge bisect with guideposts for all but the last region while (regionIndex <= stopIndex) { - byte[] currentGuidePost, endKey, endRegionKey = EMPTY_BYTE_ARRAY; + byte[] endKey, endRegionKey = EMPTY_BYTE_ARRAY; if (regionIndex == stopIndex) { endKey = stopKey; } else { @@ -510,14 +535,18 @@ private List> getParallelScans(byte[] startKey, byte[] stopKey) throw endRegionKey = regionInfo.getEndKey(); keyOffset = ScanUtil.getRowKeyOffset(regionInfo.getStartKey(), endRegionKey); } - while (guideIndex < gpsSize - && (Bytes.compareTo(currentGuidePost = gps.get(guideIndex), endKey) <= 0 || endKey.length == 0)) { - Scan newScan = scanRanges.intersectScan(scan, currentKey, currentGuidePost, keyOffset, false); - scans = addNewScan(parallelScans, scans, newScan, currentGuidePost, false, regionLocation); - currentKey = currentGuidePost; - guideIndex++; - } - Scan newScan = scanRanges.intersectScan(scan, currentKey, endKey, keyOffset, true); + try { + while (guideIndex < gpsSize + && (Bytes.compareTo(currentGuidePost.get(), + endKey) <= 0 || endKey.length == 0)) { + Scan newScan = scanRanges.intersectScan(scan, currentKey, currentGuidePost, keyOffset, false); + scans = addNewScan(parallelScans, scans, newScan, currentGuidePost.get(), false, regionLocation); + currentKey = new ImmutableBytesWritable(currentGuidePost.copyBytes()); + currentGuidePost = CodecUtils.decode(decoder, input); + guideIndex++; + } + } catch (EOFException e) {} + Scan newScan = scanRanges.intersectScan(scan, currentKey, new ImmutableBytesWritable(endKey,0,endKey.length), keyOffset, true); if (isLocalIndex) { if (newScan != null) { newScan.setAttribute(EXPECTED_UPPER_REGION_KEY, endRegionKey); @@ -526,15 +555,21 @@ private List> getParallelScans(byte[] startKey, byte[] stopKey) throw } } scans = addNewScan(parallelScans, scans, newScan, endKey, true, regionLocation); - currentKey = endKey; + currentKey = new ImmutableBytesWritable(endKey, 0, endKey.length); regionIndex++; } if (!scans.isEmpty()) { // Add any remaining scans parallelScans.add(scans); } + if (stream != null) { + try { + stream.close(); + } catch (IOException e) {} + } return parallelScans; } + public static List reverseIfNecessary(List list, boolean reverse) { if (!reverse) { return list; @@ -805,8 +840,9 @@ public void explain(List planSteps) { explain(buf.toString(),planSteps); } - @Override - public String toString() { - return "ResultIterators [name=" + getName() + ",id=" + scanId + ",scans=" + scans + "]"; - } + @Override + public String toString() { + return "ResultIterators [name=" + getName() + ",id=" + scanId + ",scans=" + scans + "]"; + } + } \ No newline at end of file diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java index 1611466451f..c16710f82d6 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java @@ -1017,26 +1017,16 @@ public static PTable createFromProto(PTableProtos.PTable table) { boolean isImmutableRows = table.getIsImmutableRows(); SortedMap tableGuidePosts = new TreeMap(Bytes.BYTES_COMPARATOR); for (PTableProtos.PTableStats pTableStatsProto : table.getGuidePostsList()) { - List value = Lists.newArrayListWithExpectedSize(pTableStatsProto.getValuesCount()); - for (int j = 0; j < pTableStatsProto.getValuesCount(); j++) { - value.add(pTableStatsProto.getValues(j).toByteArray()); - } - // No op - pTableStatsProto.getGuidePostsByteCount(); - value = Lists.newArrayListWithExpectedSize(pTableStatsProto.getValuesCount()); PGuidePosts pGuidePosts = pTableStatsProto.getPGuidePosts(); - for(int j = 0; j < pGuidePosts.getGuidePostsCount(); j++) { - value.add(pGuidePosts.getGuidePosts(j).toByteArray()); - } long guidePostsByteCount = pGuidePosts.getByteCount(); long rowCount = pGuidePosts.getRowCount(); - // TODO : Not exposing MIN/MAX key outside to client - GuidePostsInfo info = - new GuidePostsInfo(guidePostsByteCount, value, rowCount); + int maxLength = pGuidePosts.getMaxLength(); + int guidePostsCount = pGuidePosts.getGuidePostsCount(); + GuidePostsInfo info = new GuidePostsInfo(guidePostsByteCount, + new ImmutableBytesWritable(pGuidePosts.getGuidePosts().toByteArray()), rowCount, maxLength, guidePostsCount); tableGuidePosts.put(pTableStatsProto.getKey().toByteArray(), info); - } + } PTableStats stats = new PTableStatsImpl(tableGuidePosts, table.getStatsTimeStamp()); - PName dataTableName = null; if (table.hasDataTableNameBytes()) { dataTableName = PNameFactory.newName(table.getDataTableNameBytes().toByteArray()); @@ -1141,16 +1131,16 @@ public static PTableProtos.PTable toProto(PTable table) { for (Map.Entry entry : table.getTableStats().getGuidePosts().entrySet()) { PTableProtos.PTableStats.Builder statsBuilder = PTableProtos.PTableStats.newBuilder(); statsBuilder.setKey(ByteStringer.wrap(entry.getKey())); - for (byte[] stat : entry.getValue().getGuidePosts()) { - statsBuilder.addValues(ByteStringer.wrap(stat)); - } + statsBuilder.setGuidePosts(ByteStringer.wrap(entry.getValue().getGuidePosts().get())); statsBuilder.setGuidePostsByteCount(entry.getValue().getByteCount()); + statsBuilder.setMaxLength(entry.getValue().getMaxLength()); + statsBuilder.setGuidePostsCount(entry.getValue().getGuidePostsCount()); PGuidePostsProtos.PGuidePosts.Builder guidePstsBuilder = PGuidePostsProtos.PGuidePosts.newBuilder(); - for (byte[] stat : entry.getValue().getGuidePosts()) { - guidePstsBuilder.addGuidePosts(ByteStringer.wrap(stat)); - } + guidePstsBuilder.setGuidePosts(ByteStringer.wrap(entry.getValue().getGuidePosts().get())); guidePstsBuilder.setByteCount(entry.getValue().getByteCount()); guidePstsBuilder.setRowCount(entry.getValue().getRowCount()); + guidePstsBuilder.setMaxLength(entry.getValue().getMaxLength()); + guidePstsBuilder.setGuidePostsCount(entry.getValue().getGuidePostsCount()); statsBuilder.setPGuidePosts(guidePstsBuilder); builder.addGuidePosts(statsBuilder.build()); } diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsInfo.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsInfo.java index 0f1dbeb490f..00fb3499385 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsInfo.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsInfo.java @@ -17,14 +17,14 @@ */ package org.apache.phoenix.schema.stats; -import java.util.ArrayList; -import java.util.List; +import java.io.DataOutputStream; +import java.io.IOException; -import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import org.apache.hadoop.hbase.util.Bytes; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.Lists; +import org.apache.phoenix.util.ByteUtil; +import org.apache.phoenix.util.PrefixByteEncoder; +import org.apache.phoenix.util.TrustedByteArrayOutputStream; /** * A class that holds the guidePosts of a region and also allows combining the * guidePosts of different regions when the GuidePostsInfo is formed for a table. @@ -34,7 +34,7 @@ public class GuidePostsInfo { /** * the total number of guidePosts for the table combining all the guidePosts per region per cf. */ - private List guidePosts; + private ImmutableBytesWritable guidePosts; /** * The bytecount that is flattened across the total number of guide posts. */ @@ -45,7 +45,24 @@ public class GuidePostsInfo { */ private long rowCount = 0; - private long keyByteSize; // Total number of bytes in keys stored in guidePosts + /** + * Maximum length of a guidePost collected + */ + private int maxLength; + + public final static GuidePostsInfo EMPTY_GUIDEPOST = new GuidePostsInfo(0, + new ImmutableBytesWritable(ByteUtil.EMPTY_BYTE_ARRAY), 0, 0, 0); + + public int getMaxLength() { + return maxLength; + } + + private TrustedByteArrayOutputStream stream; + private PrefixByteEncoder encoder; + private DataOutputStream output; + private byte[] lastRow; + private int guidePostsCount; + private boolean isStreamInitialized; /** * Constructor that creates GuidePostsInfo per region @@ -53,22 +70,21 @@ public class GuidePostsInfo { * @param guidePosts * @param rowCount */ - public GuidePostsInfo(long byteCount, List guidePosts, long rowCount) { - this.guidePosts = ImmutableList.copyOf(guidePosts); - int size = 0; - for (byte[] key : guidePosts) { - size += key.length; - } - this.keyByteSize = size; + public GuidePostsInfo(long byteCount, ImmutableBytesWritable guidePosts, long rowCount, int maxLength, int guidePostsCount) { + this.guidePosts = guidePosts; + this.maxLength = maxLength; this.byteCount = byteCount; this.rowCount = rowCount; + this.guidePostsCount = guidePostsCount; + lastRow = ByteUtil.EMPTY_BYTE_ARRAY; } + public long getByteCount() { return byteCount; } - public List getGuidePosts() { + public ImmutableBytesWritable getGuidePosts() { return guidePosts; } @@ -79,67 +95,72 @@ public long getRowCount() { public void incrementRowCount() { this.rowCount++; } - - /** - * Combines the GuidePosts per region into one. - * @param oldInfo - */ - public void combine(GuidePostsInfo oldInfo) { - if (!oldInfo.getGuidePosts().isEmpty()) { - byte[] newFirstKey = oldInfo.getGuidePosts().get(0); - byte[] existingLastKey; - if (!this.getGuidePosts().isEmpty()) { - existingLastKey = this.getGuidePosts().get(this.getGuidePosts().size() - 1); - } else { - existingLastKey = HConstants.EMPTY_BYTE_ARRAY; - } - int size = oldInfo.getGuidePosts().size(); - // If the existing guidePosts is lesser than the new RegionInfo that we are combining - // then add the new Region info to the end of the current GuidePosts. - // If the new region info is smaller than the existing guideposts then add the existing - // guide posts after the new guideposts. - List newTotalGuidePosts = new ArrayList(this.getGuidePosts().size() + size); - if (Bytes.compareTo(existingLastKey, newFirstKey) <= 0) { - newTotalGuidePosts.addAll(this.getGuidePosts()); - newTotalGuidePosts.addAll(oldInfo.getGuidePosts()); - } else { - newTotalGuidePosts.addAll(oldInfo.getGuidePosts()); - newTotalGuidePosts.addAll(this.getGuidePosts()); - } - this.guidePosts = ImmutableList.copyOf(newTotalGuidePosts); - } - this.byteCount += oldInfo.getByteCount(); - this.keyByteSize += oldInfo.keyByteSize; - this.rowCount += oldInfo.getRowCount(); - } + public int getGuidePostsCount() { + return guidePostsCount; + } + /** * The guide posts, rowCount and byteCount are accumulated every time a guidePosts depth is * reached while collecting stats. * @param row * @param byteCount * @return + * @throws IOException */ - public boolean addGuidePost(byte[] row, long byteCount, long rowCount) { - if (guidePosts.isEmpty() || Bytes.compareTo(row, guidePosts.get(guidePosts.size() - 1)) > 0) { - List newGuidePosts = Lists.newArrayListWithExpectedSize(this.getGuidePosts().size() + 1); - newGuidePosts.addAll(guidePosts); - newGuidePosts.add(row); - this.guidePosts = ImmutableList.copyOf(newGuidePosts); - this.byteCount += byteCount; - this.keyByteSize += row.length; - this.rowCount+=rowCount; - return true; + public boolean encodeAndCollectGuidePost(byte[] row, long byteCount, long rowCount) { + if (row.length != 0 && Bytes.compareTo(lastRow, row) < 0) { + try { + if(!isStreamInitialized){ + stream = new TrustedByteArrayOutputStream(guidePosts.getLength()); + output = new DataOutputStream(stream); + stream.write(ByteUtil.copyKeyBytesIfNecessary(guidePosts)); + encoder = new PrefixByteEncoder(); + isStreamInitialized=true; + } + encoder.encode(output, row, 0, row.length); + this.byteCount += byteCount; + this.guidePostsCount++; + this.maxLength = encoder.getMaxLength(); + this.rowCount += rowCount; + lastRow = row; + return true; + } catch (IOException e) { + return false; + } } return false; } - - public boolean addGuidePost(byte[] row) { - return addGuidePost(row, 0, 0); + + public boolean encodeAndCollectGuidePost(byte[] row){ + return encodeAndCollectGuidePost(row, 0, 0); } - public boolean addGuidePost(byte[] row, long byteCount) { - return addGuidePost(row, byteCount, 0); + public boolean encodeAndCollectGuidePost(byte[] row, long byteCount){ + return encodeAndCollectGuidePost(row, byteCount, 0); } + public void close() { + if (stream != null) { + try { + stream.close(); + } catch (IOException e) { + //exception can be ignored + } + } + } + + /* + * This needs to be collect once all stats are encoded and stream buffer needs to be copied for retrieval before close() + */ + public void updateGuidePosts() { + if (stream != null) { + this.guidePosts.set(Bytes.copy(stream.getBuffer(), 0, stream.size())); + } + } + + @Override + public String toString() { + return Bytes.toString(this.guidePosts.get()); + } } \ No newline at end of file diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/PTableStatsImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/PTableStatsImpl.java index dc70e8663ac..c6b8d921918 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/PTableStatsImpl.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/PTableStatsImpl.java @@ -17,13 +17,20 @@ */ package org.apache.phoenix.schema.stats; -import java.util.List; +import java.io.ByteArrayInputStream; +import java.io.DataInput; +import java.io.DataInputStream; +import java.io.EOFException; +import java.io.IOException; import java.util.Map; import java.util.SortedMap; import java.util.TreeMap; +import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import org.apache.hadoop.hbase.util.Bytes; import org.apache.phoenix.coprocessor.MetaDataProtocol; +import org.apache.phoenix.util.CodecUtils; +import org.apache.phoenix.util.PrefixByteDecoder; import org.apache.phoenix.util.SizedUtil; import com.sun.istack.NotNull; @@ -47,12 +54,10 @@ public PTableStatsImpl(@NotNull SortedMap guidePosts, lo for (Map.Entry entry : guidePosts.entrySet()) { byte[] cf = entry.getKey(); estimatedSize += SizedUtil.ARRAY_SIZE + cf.length; - List keys = entry.getValue().getGuidePosts(); - estimatedSize += SizedUtil.sizeOfArrayList(keys.size()); - for (byte[] key : keys) { - estimatedSize += SizedUtil.ARRAY_SIZE + key.length; - } + estimatedSize += entry.getValue().getGuidePosts().getLength(); estimatedSize += SizedUtil.LONG_SIZE; + estimatedSize += SizedUtil.INT_SIZE; + estimatedSize += SizedUtil.INT_SIZE; } this.estimatedSize = estimatedSize; } @@ -65,19 +70,45 @@ public SortedMap getGuidePosts() { @Override public String toString() { StringBuilder buf = new StringBuilder(); + buf.append("PTableStats ["); for (Map.Entry entry : guidePosts.entrySet()) { buf.append(Bytes.toStringBinary(entry.getKey())); buf.append(":("); - List keys = entry.getValue().getGuidePosts(); - if (!keys.isEmpty()) { - for (byte[] key : keys) { - buf.append(Bytes.toStringBinary(key)); - buf.append(","); + ImmutableBytesWritable keys = entry.getValue().getGuidePosts(); + ByteArrayInputStream stream = new ByteArrayInputStream(keys.get(), keys.getOffset(), keys.getLength()); + try { + if (keys.getLength() != 0) { + DataInput input = new DataInputStream(stream); + PrefixByteDecoder decoder; + if (entry.getValue().getMaxLength() > 0) { + decoder = new PrefixByteDecoder(entry.getValue().getMaxLength()); + } else { + decoder = new PrefixByteDecoder(); + } + try { + while (true) { + ImmutableBytesWritable ptr = CodecUtils.decode(decoder, input); + buf.append(Bytes.toStringBinary(ptr.get())); + buf.append(","); + } + } catch (EOFException e) { // Ignore as this signifies we're done + + } finally { + try { + stream.close(); + } catch (IOException e) {} + } + buf.setLength(buf.length() - 1); + } + buf.append(")"); + } finally { + if (stream != null) { + try { + stream.close(); + } catch (IOException e) {} } - buf.setLength(buf.length()-1); } - buf.append(")"); } buf.append("]"); return buf.toString(); diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollector.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollector.java index fc8d8bddb4a..e51e2bdca4d 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollector.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollector.java @@ -19,7 +19,6 @@ import java.io.IOException; import java.util.ArrayList; -import java.util.Collections; import java.util.List; import java.util.Map; @@ -99,7 +98,7 @@ public StatisticsCollector(RegionCoprocessorEnvironment env, String tableName, l // in a compaction we know the one family ahead of time if (family != null) { ImmutableBytesPtr cfKey = new ImmutableBytesPtr(family); - cachedGps = new Pair(0l, new GuidePostsInfo(0, Collections. emptyList(), 0l)); + cachedGps = new Pair(0l, new GuidePostsInfo(0, new ImmutableBytesWritable(ByteUtil.EMPTY_BYTE_ARRAY), 0l, 0, 0)); guidePostsMap.put(cfKey, cachedGps); } } @@ -110,6 +109,12 @@ public long getMaxTimeStamp() { public void close() throws IOException { this.statsTable.close(); + for (ImmutableBytesPtr ptr : guidePostsMap.keySet()) { + GuidePostsInfo guidePostInfo = guidePostsMap.get(ptr).getSecond(); + if (guidePostInfo != null) { + guidePostInfo.close(); + } + } } public void updateStatistic(Region region) { @@ -175,7 +180,7 @@ public void collectStatistics(final List results) { gps = guidePostsMap.get(cfKey); if (gps == null) { gps = new Pair(0l, - new GuidePostsInfo(0, Collections. emptyList(), 0l)); + new GuidePostsInfo(0,new ImmutableBytesWritable(ByteUtil.EMPTY_BYTE_ARRAY), 0l, 0, 0)); guidePostsMap.put(cfKey, gps); } if (famMap.get(cfKey) == null) { @@ -191,7 +196,7 @@ public void collectStatistics(final List results) { if (byteCount >= guidepostDepth) { byte[] row = ByteUtil.copyKeyBytesIfNecessary( new ImmutableBytesWritable(kv.getRowArray(), kv.getRowOffset(), kv.getRowLength())); - if (gps.getSecond().addGuidePost(row, byteCount)) { + if (gps.getSecond().encodeAndCollectGuidePost(row, byteCount)) { gps.setFirst(0l); } } @@ -225,21 +230,19 @@ public void clear() { maxTimeStamp = MetaDataProtocol.MIN_TABLE_TIMESTAMP; } - public void addGuidePost(ImmutableBytesPtr cfKey, GuidePostsInfo info, long byteSize, long timestamp, - byte[] minKey) { - Pair newInfo = new Pair(byteSize, info); - Pair oldInfo = guidePostsMap.put(cfKey, newInfo); - if (oldInfo != null) { - info.combine(oldInfo.getSecond()); - newInfo.setFirst(oldInfo.getFirst() + newInfo.getFirst()); - } - maxTimeStamp = Math.max(maxTimeStamp, timestamp); - } - public GuidePostsInfo getGuidePosts(ImmutableBytesPtr fam) { Pair pair = guidePostsMap.get(fam); if (pair != null) { return pair.getSecond(); } return null; } + public void updateGuidePosts() { + for (ImmutableBytesPtr cf : guidePostsMap.keySet()) { + GuidePostsInfo guidePostInfo = guidePostsMap.get(cf).getSecond(); + if (guidePostInfo != null) { + guidePostInfo.updateGuidePosts(); + } + } + } + } diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsScanner.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsScanner.java index d6f0bf13571..ffce33674b3 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsScanner.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsScanner.java @@ -93,6 +93,7 @@ protected void updateStat(final List results) { @Override public void close() throws IOException { boolean async = config.getBoolean(COMMIT_STATS_ASYNC, DEFAULT_COMMIT_STATS_ASYNC); + tracker.updateGuidePosts(); StatisticsCollectionRunTracker collectionTracker = StatisticsCollectionRunTracker.getInstance(config); StatisticsScannerCallable callable = new StatisticsScannerCallable(); if (!async) { @@ -134,7 +135,8 @@ public Void call() throws IOException { } finally { try { collectionTracker.removeCompactingRegion(regionInfo); - stats.close(); + stats.close();// close the writer + tracker.close();// close the tracker } catch (IOException e) { if (toThrow == null) toThrow = e; LOG.error("Error while closing the stats table", e); diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsUtil.java index 16f249c6ecd..5ea7462d140 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsUtil.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsUtil.java @@ -21,7 +21,6 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collections; import java.util.List; import java.util.TreeMap; @@ -55,10 +54,14 @@ private StatisticsUtil() { /** Number of parts in our complex key */ protected static final int NUM_KEY_PARTS = 3; - + public static byte[] getRowKey(byte[] table, ImmutableBytesPtr fam, byte[] guidePostStartKey) { + return getRowKey(table, fam, new ImmutableBytesWritable(guidePostStartKey,0,guidePostStartKey.length)); + } + + public static byte[] getRowKey(byte[] table, ImmutableBytesPtr fam, ImmutableBytesWritable guidePostStartKey) { // always starts with the source table - byte[] rowKey = new byte[table.length + fam.getLength() + guidePostStartKey.length + 2]; + byte[] rowKey = new byte[table.length + fam.getLength() + guidePostStartKey.getLength() + 2]; int offset = 0; System.arraycopy(table, 0, rowKey, offset, table.length); offset += table.length; @@ -66,7 +69,7 @@ public static byte[] getRowKey(byte[] table, ImmutableBytesPtr fam, byte[] guide System.arraycopy(fam.get(), fam.getOffset(), rowKey, offset, fam.getLength()); offset += fam.getLength(); rowKey[offset++] = QueryConstants.SEPARATOR_BYTE; // assumes stats table columns not DESC - System.arraycopy(guidePostStartKey, 0, rowKey, offset, guidePostStartKey.length); + System.arraycopy(guidePostStartKey.get(), 0, rowKey, offset, guidePostStartKey.getLength()); return rowKey; } @@ -170,16 +173,29 @@ public static PTableStats readStatistics(HTableInterface statsHTable, byte[] tab byte[] newGPStartKey = getGuidePostsInfoFromRowKey(tableNameBytes, cfName, result.getRow()); GuidePostsInfo guidePosts = guidePostsPerCf.get(cfName); if (guidePosts == null) { - guidePosts = new GuidePostsInfo(0l, Collections. emptyList(), 0l); + guidePosts = new GuidePostsInfo(0l, new ImmutableBytesWritable(ByteUtil.EMPTY_BYTE_ARRAY), 0l, + 0, 0); guidePostsPerCf.put(cfName, guidePosts); } - guidePosts.addGuidePost(newGPStartKey, byteCount, rowCount); + guidePosts.encodeAndCollectGuidePost(newGPStartKey, byteCount, rowCount); + } + } + for (byte[] cf : guidePostsPerCf.keySet()) { + GuidePostsInfo guidePostInfo = guidePostsPerCf.get(cf); + if (guidePostInfo != null) { + guidePostInfo.updateGuidePosts(); } } } finally { if (scanner != null) { scanner.close(); } + for (byte[] cf : guidePostsPerCf.keySet()) { + GuidePostsInfo guidePostInfo = guidePostsPerCf.get(cf); + if (guidePostInfo != null) { + guidePostInfo.close(); + } + } } if (!guidePostsPerCf.isEmpty()) { return new PTableStatsImpl(guidePostsPerCf, timeStamp); } return PTableStats.EMPTY_STATS; diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsWriter.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsWriter.java index b8bd06450b2..1dcfb3df265 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsWriter.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsWriter.java @@ -17,7 +17,11 @@ */ package org.apache.phoenix.schema.stats; +import java.io.ByteArrayInputStream; import java.io.Closeable; +import java.io.DataInput; +import java.io.DataInputStream; +import java.io.EOFException; import java.io.IOException; import java.sql.Date; import java.util.List; @@ -45,6 +49,7 @@ import org.apache.phoenix.schema.types.PDate; import org.apache.phoenix.schema.types.PLong; import org.apache.phoenix.util.ByteUtil; +import org.apache.phoenix.util.PrefixByteDecoder; import org.apache.phoenix.util.ServerUtil; import org.apache.phoenix.util.TimeKeeper; @@ -103,6 +108,7 @@ private StatisticsWriter(HTableInterface statsReaderTable, HTableInterface stats @Override public void close() throws IOException { statsWriterTable.close(); + statsReaderTable.close(); } /** @@ -134,21 +140,36 @@ public void addStats(StatisticsCollector tracker, ImmutableBytesPtr cfKey, List< GuidePostsInfo gps = tracker.getGuidePosts(cfKey); if (gps != null) { boolean rowColumnAdded = false; - for (byte[] gp : gps.getGuidePosts()) { - byte[] prefix = StatisticsUtil.getRowKey(tableName, cfKey, gp); - Put put = new Put(prefix); - if (!rowColumnAdded) { - put.add(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES, PhoenixDatabaseMetaData.GUIDE_POSTS_WIDTH_BYTES, - timeStamp, PLong.INSTANCE.toBytes(gps.getByteCount())); - put.add(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES, - PhoenixDatabaseMetaData.GUIDE_POSTS_ROW_COUNT_BYTES, timeStamp, - PLong.INSTANCE.toBytes(gps.getRowCount())); - rowColumnAdded = true; + ImmutableBytesWritable keys = gps.getGuidePosts(); + ByteArrayInputStream stream = new ByteArrayInputStream(keys.get(), keys.getOffset(), keys.getLength()); + DataInput input = new DataInputStream(stream); + PrefixByteDecoder decoder = new PrefixByteDecoder(); + try { + while (true) { + ImmutableBytesWritable ptr = decoder.decode(input); + byte[] prefix = StatisticsUtil.getRowKey(tableName, cfKey, ptr); + Put put = new Put(prefix); + if (!rowColumnAdded) { + put.add(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES, PhoenixDatabaseMetaData.GUIDE_POSTS_WIDTH_BYTES, + timeStamp, PLong.INSTANCE.toBytes(gps.getByteCount())); + put.add(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES, + PhoenixDatabaseMetaData.GUIDE_POSTS_ROW_COUNT_BYTES, timeStamp, + PLong.INSTANCE.toBytes(gps.getRowCount())); + rowColumnAdded = true; + } + // Add our empty column value so queries behave correctly + put.add(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES, QueryConstants.EMPTY_COLUMN_BYTES, timeStamp, + ByteUtil.EMPTY_BYTE_ARRAY); + mutations.add(put); } - // Add our empty column value so queries behave correctly - put.add(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES, QueryConstants.EMPTY_COLUMN_BYTES, timeStamp, - ByteUtil.EMPTY_BYTE_ARRAY); - mutations.add(put); + } catch (EOFException e) { // Ignore as this signifies we're done + + } finally { + try { + if (stream != null) { + stream.close(); + } + } catch (IOException e) {} } } } diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/ByteUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/ByteUtil.java index 64d064a75d9..44502a5478e 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/util/ByteUtil.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/util/ByteUtil.java @@ -49,6 +49,8 @@ public class ByteUtil { public static final byte[] EMPTY_BYTE_ARRAY = new byte[0]; public static final ImmutableBytesPtr EMPTY_BYTE_ARRAY_PTR = new ImmutableBytesPtr( EMPTY_BYTE_ARRAY); + public static final ImmutableBytesWritable EMPTY_IMMUTABLE_BYTE_ARRAY = new ImmutableBytesWritable( + EMPTY_BYTE_ARRAY); public static final Comparator BYTES_PTR_COMPARATOR = new Comparator() { diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/CodecUtils.java b/phoenix-core/src/main/java/org/apache/phoenix/util/CodecUtils.java new file mode 100644 index 00000000000..cc160bb53a0 --- /dev/null +++ b/phoenix-core/src/main/java/org/apache/phoenix/util/CodecUtils.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.util; + +import java.io.ByteArrayInputStream; +import java.io.DataInput; +import java.io.DataInputStream; +import java.io.DataOutput; +import java.io.DataOutputStream; +import java.io.EOFException; +import java.io.IOException; +import java.util.List; + +import org.apache.hadoop.hbase.io.ImmutableBytesWritable; + +import com.google.common.collect.Lists; + +public class CodecUtils { + + public static List decodeBytes(ImmutableBytesWritable encodedBytes, int maxLength) throws IOException { + ByteArrayInputStream stream = new ByteArrayInputStream(encodedBytes.get(), encodedBytes.getOffset(), encodedBytes.getLength()); + DataInput input = new DataInputStream(stream); + PrefixByteDecoder decoder; + if (maxLength > 0) { + decoder = new PrefixByteDecoder(maxLength); + } else { + decoder = new PrefixByteDecoder(); + } + List listOfBytes = Lists.newArrayList(); + try { + while (true) { + ImmutableBytesWritable ptr = decoder.decode(input); + // For this test, copy the bytes, but we wouldn't do this unless + // necessary for non testing + listOfBytes.add(ptr.copyBytes()); + } + } catch (EOFException e) { // Ignore as this signifies we're done + + } + return listOfBytes; + } + + public static int encodeBytes(List listOfBytes, ImmutableBytesWritable ptr) throws IOException { + TrustedByteArrayOutputStream stream = new TrustedByteArrayOutputStream(calculateSize(listOfBytes)); + DataOutput output = new DataOutputStream(stream); + PrefixByteEncoder encoder = new PrefixByteEncoder(); + for (byte[] bytes : listOfBytes) { + encoder.encode(output, bytes, 0, bytes.length); + } + stream.close(); + ptr.set(stream.getBuffer(), 0, stream.size()); + return encoder.getMaxLength(); + } + + public static int calculateSize(List listOfBytes) { + int size = 0; + for (byte[] bytes : listOfBytes) { + size += bytes.length; + } + return size; + } + + public static ImmutableBytesWritable decode(PrefixByteDecoder decoder, DataInput input) throws EOFException { + try { + ImmutableBytesWritable val= decoder.decode(input); + return val; + } catch(EOFException eof){ + throw eof; + }catch (IOException e) { + throw new RuntimeException(e); + } + } + +} diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/PrefixByteDecoder.java b/phoenix-core/src/main/java/org/apache/phoenix/util/PrefixByteDecoder.java new file mode 100644 index 00000000000..420ac83d9eb --- /dev/null +++ b/phoenix-core/src/main/java/org/apache/phoenix/util/PrefixByteDecoder.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.util; + +import java.io.DataInput; +import java.io.IOException; + +import org.apache.hadoop.hbase.io.ImmutableBytesWritable; +import org.apache.hadoop.io.WritableUtils; + +/** + * + * Prefix decoder for byte arrays. For encoding, see {@link PrefixByteEncoder}. + * + */ +public class PrefixByteDecoder { + private final int maxLength; + private final ImmutableBytesWritable previous; + + /** + * Used when the maximum length of encoded byte array is not known. Will + * cause a new byte array to be allocated for each call to {@link #decode(DataInput)}. + */ + public PrefixByteDecoder() { + maxLength = -1; + previous = new ImmutableBytesWritable(ByteUtil.EMPTY_BYTE_ARRAY); + } + + /** + * Used when the maximum length of encoded byte array is known in advance. + * Will not allocate new byte array with each call to {@link #decode(DataInput)}. + * @param maxLength maximum length needed for any call to {@link #decode(DataInput)}. + */ + public PrefixByteDecoder(int maxLength) { + this.maxLength = maxLength; + this.previous = new ImmutableBytesWritable(new byte[maxLength],0,0); + } + + /** + * Resets state of decoder if it will be used to decode bytes from a + * different DataInput. + */ + public void reset() { + previous.set(previous.get(),0,0); + } + + /** + * Decodes bytes encoded with {@link PrefixByteEncoder}. + * @param in Input from which bytes are read. + * @return Pointer containing bytes that were decoded. Note that the + * same pointer will be returned with each call, so it must be consumed + * prior to calling decode again. + * @throws IOException + */ + public ImmutableBytesWritable decode(DataInput in) throws IOException { + int prefixLen = WritableUtils.readVInt(in); + int suffixLen = WritableUtils.readVInt(in); + int length = prefixLen + suffixLen; + byte[] b; + if (maxLength == -1) { // Allocate new byte array each time + b = new byte[length]; + System.arraycopy(previous.get(), previous.getOffset(), b, 0, prefixLen); + } else { // Reuse same buffer each time + b = previous.get(); + } + in.readFully(b, prefixLen, suffixLen); + previous.set(b, 0, length); + return previous; + } +} diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/PrefixByteEncoder.java b/phoenix-core/src/main/java/org/apache/phoenix/util/PrefixByteEncoder.java new file mode 100644 index 00000000000..7a04aa0b4a6 --- /dev/null +++ b/phoenix-core/src/main/java/org/apache/phoenix/util/PrefixByteEncoder.java @@ -0,0 +1,99 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.util; + +import java.io.DataOutput; +import java.io.IOException; + +import org.apache.hadoop.hbase.io.ImmutableBytesWritable; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.io.WritableUtils; + +/** + * + * Prefix encoder for byte arrays. For decoding, see {@link PrefixByteDecoder}. + * + */ +public class PrefixByteEncoder { + private int maxLength; + private final ImmutableBytesWritable previous = new ImmutableBytesWritable(ByteUtil.EMPTY_BYTE_ARRAY); + + public PrefixByteEncoder() { + } + + /** + * Resets the state of the encoder to its initial state (i.e. forgetting + * the previous byte array that may have been encoded). + */ + public void reset() { + previous.set(ByteUtil.EMPTY_BYTE_ARRAY); + } + + /** + * @return the maximum length byte array encountered while encoding + */ + public int getMaxLength() { + return maxLength; + } + + /** + * Prefix encodes the byte array pointed to into the output stream + * @param out output stream to encode into + * @param ptr pointer to byte array to encode. + * @throws IOException + */ + public void encode(DataOutput out, ImmutableBytesWritable ptr) throws IOException { + encode(out, ptr.get(), ptr.getOffset(), ptr.getLength()); + } + + /** + * Prefix encodes the byte array into the output stream + * @param out output stream to encode into + * @param b byte array to encode + * @throws IOException + */ + public void encode(DataOutput out, byte[] b) throws IOException { + encode(out, b, 0, b.length); + } + + /** + * Prefix encodes the byte array from offset to length into output stream. + * Instead of writing the entire byte array, only the portion of the byte array + * that differs from the beginning of the previous byte array written is written. + * + * @param out output stream to encode into + * @param b byte array buffer + * @param offset offset into byte array to start encoding + * @param length length of byte array to encode + * @throws IOException + */ + public void encode(DataOutput out, byte[] b, int offset, int length) throws IOException { + int i = 0; + int prevOffset = previous.getOffset(); + byte[] prevBytes = previous.get(); + int prevLength = previous.getLength(); + int minLength = prevLength < b.length ? prevLength : b.length; + for(i = 0; (i < minLength) && (prevBytes[prevOffset + i] == b[offset + i]); i++); + WritableUtils.writeVInt(out, i); + Bytes.writeByteArray(out, b, offset + i, length - i); + previous.set(b, offset, length); + if (length > maxLength) { + maxLength = length; + } + } +} diff --git a/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanBigFilterTest.java b/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanBigFilterTest.java index a0aca4dd18c..b30f8f0505d 100644 --- a/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanBigFilterTest.java +++ b/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanBigFilterTest.java @@ -22,12 +22,11 @@ import java.sql.Connection; import java.sql.DriverManager; import java.sql.PreparedStatement; -import java.util.Collections; import java.util.Map; import java.util.Properties; import java.util.SortedMap; -import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import org.apache.hadoop.hbase.util.Bytes; import org.apache.phoenix.compile.QueryPlan; import org.apache.phoenix.end2end.Shadower; @@ -41,6 +40,7 @@ import org.apache.phoenix.schema.PTableKey; import org.apache.phoenix.schema.stats.GuidePostsInfo; import org.apache.phoenix.schema.stats.PTableStats; +import org.apache.phoenix.util.ByteUtil; import org.apache.phoenix.util.PropertiesUtil; import org.apache.phoenix.util.ReadOnlyProps; import org.junit.BeforeClass; @@ -648,10 +648,12 @@ private void testIntersect(byte[][] regionBoundaries, byte[][] guidePosts) throw stmt.execute(); final PTable table = conn.unwrap(PhoenixConnection.class).getTable(new PTableKey(null, "PERF.BIG_OLAP_DOC")); - GuidePostsInfo info = new GuidePostsInfo(0,Collections. emptyList(), 0l); + GuidePostsInfo info = new GuidePostsInfo(0, new ImmutableBytesWritable(ByteUtil.EMPTY_BYTE_ARRAY), 0l, 0, 0); for (byte[] gp : guidePosts) { - info.addGuidePost(gp, 1000); + info.encodeAndCollectGuidePost(gp, 1000); } + info.updateGuidePosts(); + info.close(); final SortedMap gpMap = Maps.newTreeMap(Bytes.BYTES_COMPARATOR); gpMap.put(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES, info); PTable tableWithStats = PTableImpl.makePTable(table, new PTableStats() { diff --git a/phoenix-core/src/test/java/org/apache/phoenix/util/PrefixByteEncoderDecoderTest.java b/phoenix-core/src/test/java/org/apache/phoenix/util/PrefixByteEncoderDecoderTest.java new file mode 100644 index 00000000000..ecb3efff94c --- /dev/null +++ b/phoenix-core/src/test/java/org/apache/phoenix/util/PrefixByteEncoderDecoderTest.java @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.util; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.io.DataOutput; +import java.io.DataOutputStream; +import java.io.IOException; +import java.util.Arrays; +import java.util.List; + +import org.apache.hadoop.hbase.io.ImmutableBytesWritable; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.io.WritableUtils; +import org.apache.phoenix.query.QueryConstants; +import org.junit.Test; + + +public class PrefixByteEncoderDecoderTest { + + static final List guideposts = Arrays.asList( + ByteUtil.concat(Bytes.toBytes("aaaaaaaaaa"), QueryConstants.SEPARATOR_BYTE_ARRAY, Bytes.toBytes(1000L), Bytes.toBytes("bbbbbbbbbb")), + ByteUtil.concat(Bytes.toBytes("aaaaaaaaaa"), QueryConstants.SEPARATOR_BYTE_ARRAY, Bytes.toBytes(1000L), Bytes.toBytes("bbbbbccccc")), + ByteUtil.concat(Bytes.toBytes("aaaaaaaaaa"), QueryConstants.SEPARATOR_BYTE_ARRAY, Bytes.toBytes(2000L), Bytes.toBytes("bbbbbbbbbb")), + ByteUtil.concat(Bytes.toBytes("bbbbbbbbbb"), QueryConstants.SEPARATOR_BYTE_ARRAY, Bytes.toBytes(1000L), Bytes.toBytes("bbbbbbbbbb")), + ByteUtil.concat(Bytes.toBytes("bbbbbbbbbb"), QueryConstants.SEPARATOR_BYTE_ARRAY, Bytes.toBytes(2000L), Bytes.toBytes("bbbbbbbbbb")), + ByteUtil.concat(Bytes.toBytes("bbbbbbbbbb"), QueryConstants.SEPARATOR_BYTE_ARRAY, Bytes.toBytes(2000L), Bytes.toBytes("c")), + ByteUtil.concat(Bytes.toBytes("bbbbbbbbbbb"), QueryConstants.SEPARATOR_BYTE_ARRAY, Bytes.toBytes(1000L), Bytes.toBytes("bbbbbbbbbb")), + ByteUtil.concat(Bytes.toBytes("d"), QueryConstants.SEPARATOR_BYTE_ARRAY, Bytes.toBytes(1000L), Bytes.toBytes("bbbbbbbbbb")), + ByteUtil.concat(Bytes.toBytes("d"), QueryConstants.SEPARATOR_BYTE_ARRAY, Bytes.toBytes(1000L), Bytes.toBytes("bbbbbbbbbbc")), + ByteUtil.concat(Bytes.toBytes("e"), QueryConstants.SEPARATOR_BYTE_ARRAY, Bytes.toBytes(1000L), Bytes.toBytes("bbbbbbbbbb")) + ); + + @Test + public void testEncode() throws IOException { + List listOfBytes = Arrays.asList(Bytes.toBytes("aaaaa"), Bytes.toBytes("aaaabb")); + ImmutableBytesWritable ptr = new ImmutableBytesWritable(); + int maxLength = CodecUtils.encodeBytes(listOfBytes, ptr); + assertEquals(6, maxLength); + TrustedByteArrayOutputStream stream = new TrustedByteArrayOutputStream(CodecUtils.calculateSize(listOfBytes)); + DataOutput output = new DataOutputStream(stream); + WritableUtils.writeVInt(output, 0); + WritableUtils.writeVInt(output, 5); + output.write(Bytes.toBytes("aaaaa")); // No space savings on first key + WritableUtils.writeVInt(output, 4); + WritableUtils.writeVInt(output, 2); + output.write(Bytes.toBytes("bb")); // Only writes part of second key that's different + assertArrayEquals(stream.toByteArray(), ptr.copyBytes()); + } + + @Test + public void testEncodeDecodeWithSingleBuffer() throws IOException { + testEncodeDecode(true); + } + + @Test + public void testEncodeDecodeWithNewBuffer() throws IOException { + testEncodeDecode(false); + } + + private void testEncodeDecode(boolean useSingleBuffer) throws IOException { + ImmutableBytesWritable ptr = new ImmutableBytesWritable(); + int maxLength = CodecUtils.encodeBytes(guideposts, ptr); + int encodedSize = ptr.getLength(); + int unencodedSize = CodecUtils.calculateSize(guideposts); + assertTrue(encodedSize < unencodedSize); + List listOfBytes = CodecUtils.decodeBytes(ptr, useSingleBuffer ? maxLength : -1); + assertListByteArraysEquals(guideposts, listOfBytes); + } + + private static void assertListByteArraysEquals(List listOfBytes1, List listOfBytes2) { + assertEquals(listOfBytes1.size(), listOfBytes2.size()); + for (int i = 0; i < listOfBytes1.size(); i++) { + assertArrayEquals(listOfBytes1.get(i), listOfBytes2.get(i)); + } + } + +} diff --git a/phoenix-protocol/src/main/PGuidePosts.proto b/phoenix-protocol/src/main/PGuidePosts.proto index 047a658de3e..d0c7c9bdfd7 100644 --- a/phoenix-protocol/src/main/PGuidePosts.proto +++ b/phoenix-protocol/src/main/PGuidePosts.proto @@ -24,7 +24,9 @@ option java_generic_services = true; option java_generate_equals_and_hash = true; option optimize_for = SPEED; message PGuidePosts { - repeated bytes guidePosts = 1; + optional bytes guidePosts = 1; optional int64 byteCount = 2; optional int64 rowCount = 3; + optional int32 maxLength = 4; + optional int32 guidePostsCount = 5; } \ No newline at end of file diff --git a/phoenix-protocol/src/main/PTable.proto b/phoenix-protocol/src/main/PTable.proto index 485dd11cdf9..d7b407bba6c 100644 --- a/phoenix-protocol/src/main/PTable.proto +++ b/phoenix-protocol/src/main/PTable.proto @@ -52,11 +52,12 @@ message PColumn { message PTableStats { required bytes key = 1; - repeated bytes values = 2; + optional bytes guidePosts = 2; optional int64 guidePostsByteCount = 3; optional int64 keyBytesCount = 4; optional int32 guidePostsCount = 5; optional PGuidePosts pGuidePosts = 6; + optional int32 maxLength = 7; } message PTable { From 38f1676ba9b572d234567d87eaa0b7db461defc5 Mon Sep 17 00:00:00 2001 From: Ankit Singhal Date: Sat, 16 Jan 2016 02:22:29 +0530 Subject: [PATCH 34/47] PHOENIX-2417 Review comments --- .../apache/phoenix/compile/ScanRanges.java | 10 +++--- .../phoenix/iterate/BaseResultIterators.java | 36 ++++++++----------- .../org/apache/phoenix/schema/PTableImpl.java | 3 +- .../phoenix/schema/stats/GuidePostsInfo.java | 11 ++---- .../phoenix/schema/stats/PTableStatsImpl.java | 18 ++-------- .../schema/stats/StatisticsWriter.java | 9 ++--- .../org/apache/phoenix/util/CodecUtils.java | 31 +++++++++++----- .../phoenix/util/PrefixByteDecoder.java | 9 +++-- .../compile/ScanRangesIntersectTest.java | 4 ++- 9 files changed, 62 insertions(+), 69 deletions(-) diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/ScanRanges.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/ScanRanges.java index 91d07ac898d..4d343f342e8 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/compile/ScanRanges.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/ScanRanges.java @@ -237,9 +237,7 @@ private static byte[] stripPrefix(byte[] key, int keyOffset) { return temp; } - public Scan intersectScan(Scan scan, final ImmutableBytesWritable originalStartKeyPtr, final ImmutableBytesWritable originalStopKeyPtr, final int keyOffset, boolean crossesRegionBoundary) { - byte[] originalStartKey= originalStartKeyPtr.get(); - byte[] originalStopKey= originalStopKeyPtr.get(); + public Scan intersectScan(Scan scan, final byte[] originalStartKey, final byte[] originalStopKey, final int keyOffset, boolean crossesRegionBoundary) { byte[] startKey = originalStartKey; byte[] stopKey = originalStopKey; if (stopKey.length > 0 && Bytes.compareTo(startKey, stopKey) >= 0) { @@ -386,8 +384,8 @@ public Scan intersectScan(Scan scan, final ImmutableBytesWritable originalStartK if (scanStopKey.length > 0 && Bytes.compareTo(scanStartKey, scanStopKey) >= 0) { return null; } - newScan.setStartRow(Bytes.copy(scanStartKey)); - newScan.setStopRow(Bytes.copy(scanStopKey)); + newScan.setStartRow(scanStartKey); + newScan.setStopRow(scanStopKey); if(keyOffset > 0) { newScan.setAttribute(STARTKEY_OFFSET, Bytes.toBytes(keyOffset)); } @@ -418,7 +416,7 @@ public boolean intersects(byte[] lowerInclusiveKey, byte[] upperExclusiveKey, in } //return filter.hasIntersect(lowerInclusiveKey, upperExclusiveKey); - return intersectScan(null, new ImmutableBytesWritable(lowerInclusiveKey), new ImmutableBytesWritable(upperExclusiveKey), keyOffset, crossesRegionBoundary) == HAS_INTERSECTION; + return intersectScan(null, lowerInclusiveKey, upperExclusiveKey, keyOffset, crossesRegionBoundary) == HAS_INTERSECTION; } public SkipScanFilter getSkipScanFilter() { diff --git a/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java b/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java index 1a5cdc55cfb..e17ce86bf08 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java @@ -26,7 +26,6 @@ import java.io.DataInput; import java.io.DataInputStream; import java.io.EOFException; -import java.io.IOException; import java.sql.SQLException; import java.util.ArrayList; import java.util.Collections; @@ -508,21 +507,18 @@ private List> getParallelScans(byte[] startKey, byte[] stopKey) throw if (gpsSize > 0) { stream = new ByteArrayInputStream(guidePosts.get(), guidePosts.getOffset(), guidePosts.getLength()); input = new DataInputStream(stream); - if (gps.getMaxLength() > 0) { - decoder = new PrefixByteDecoder(gps.getMaxLength()); - } else { - decoder = new PrefixByteDecoder(); - } + decoder = new PrefixByteDecoder(gps.getMaxLength()); try { - currentGuidePost = CodecUtils.decode(decoder, input); - while (currentKey.getLength() != 0 && Bytes.compareTo(currentKey.get(), currentGuidePost.get()) <= 0) { - currentGuidePost = CodecUtils.decode(decoder, input); + while (currentKey.compareTo(currentGuidePost = CodecUtils.decode(decoder, input)) >= 0) { guideIndex++; } } catch (EOFException e) {} } + byte[] currentKeyBytes = ByteUtil.copyKeyBytesIfNecessary(currentKey); + // Merge bisect with guideposts for all but the last region while (regionIndex <= stopIndex) { + byte[] currentGuidePostBytes = currentGuidePost.copyBytes(); byte[] endKey, endRegionKey = EMPTY_BYTE_ARRAY; if (regionIndex == stopIndex) { endKey = stopKey; @@ -536,17 +532,17 @@ private List> getParallelScans(byte[] startKey, byte[] stopKey) throw keyOffset = ScanUtil.getRowKeyOffset(regionInfo.getStartKey(), endRegionKey); } try { - while (guideIndex < gpsSize - && (Bytes.compareTo(currentGuidePost.get(), - endKey) <= 0 || endKey.length == 0)) { - Scan newScan = scanRanges.intersectScan(scan, currentKey, currentGuidePost, keyOffset, false); - scans = addNewScan(parallelScans, scans, newScan, currentGuidePost.get(), false, regionLocation); - currentKey = new ImmutableBytesWritable(currentGuidePost.copyBytes()); + while (guideIndex < gpsSize && (currentGuidePost.compareTo(endKey) <= 0 || endKey.length == 0)) { + Scan newScan = scanRanges.intersectScan(scan, currentKeyBytes, currentGuidePostBytes, keyOffset, + false); + scans = addNewScan(parallelScans, scans, newScan, currentGuidePostBytes, false, regionLocation); + currentKeyBytes = currentGuidePost.copyBytes(); currentGuidePost = CodecUtils.decode(decoder, input); + currentGuidePostBytes = ByteUtil.copyKeyBytesIfNecessary(currentGuidePost); guideIndex++; } } catch (EOFException e) {} - Scan newScan = scanRanges.intersectScan(scan, currentKey, new ImmutableBytesWritable(endKey,0,endKey.length), keyOffset, true); + Scan newScan = scanRanges.intersectScan(scan, currentKeyBytes, endKey, keyOffset, true); if (isLocalIndex) { if (newScan != null) { newScan.setAttribute(EXPECTED_UPPER_REGION_KEY, endRegionKey); @@ -555,17 +551,13 @@ private List> getParallelScans(byte[] startKey, byte[] stopKey) throw } } scans = addNewScan(parallelScans, scans, newScan, endKey, true, regionLocation); - currentKey = new ImmutableBytesWritable(endKey, 0, endKey.length); + currentKeyBytes = endKey; regionIndex++; } if (!scans.isEmpty()) { // Add any remaining scans parallelScans.add(scans); } - if (stream != null) { - try { - stream.close(); - } catch (IOException e) {} - } + CodecUtils.close(stream); return parallelScans; } diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java index c16710f82d6..7df5b1c5e9f 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java @@ -75,6 +75,7 @@ import com.google.common.collect.ListMultimap; import com.google.common.collect.Lists; import com.google.common.collect.Maps; +import com.google.protobuf.HBaseZeroCopyByteString; import com.sun.istack.NotNull; import co.cask.tephra.TxConstants; @@ -1023,7 +1024,7 @@ public static PTable createFromProto(PTableProtos.PTable table) { int maxLength = pGuidePosts.getMaxLength(); int guidePostsCount = pGuidePosts.getGuidePostsCount(); GuidePostsInfo info = new GuidePostsInfo(guidePostsByteCount, - new ImmutableBytesWritable(pGuidePosts.getGuidePosts().toByteArray()), rowCount, maxLength, guidePostsCount); + new ImmutableBytesWritable(HBaseZeroCopyByteString.zeroCopyGetBytes(pGuidePosts.getGuidePosts())), rowCount, maxLength, guidePostsCount); tableGuidePosts.put(pTableStatsProto.getKey().toByteArray(), info); } PTableStats stats = new PTableStatsImpl(tableGuidePosts, table.getStatsTimeStamp()); diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsInfo.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsInfo.java index 00fb3499385..b7f5c49f87c 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsInfo.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsInfo.java @@ -23,6 +23,7 @@ import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import org.apache.hadoop.hbase.util.Bytes; import org.apache.phoenix.util.ByteUtil; +import org.apache.phoenix.util.CodecUtils; import org.apache.phoenix.util.PrefixByteEncoder; import org.apache.phoenix.util.TrustedByteArrayOutputStream; /** @@ -71,7 +72,7 @@ public int getMaxLength() { * @param rowCount */ public GuidePostsInfo(long byteCount, ImmutableBytesWritable guidePosts, long rowCount, int maxLength, int guidePostsCount) { - this.guidePosts = guidePosts; + this.guidePosts = new ImmutableBytesWritable(guidePosts); this.maxLength = maxLength; this.byteCount = byteCount; this.rowCount = rowCount; @@ -141,13 +142,7 @@ public boolean encodeAndCollectGuidePost(byte[] row, long byteCount){ } public void close() { - if (stream != null) { - try { - stream.close(); - } catch (IOException e) { - //exception can be ignored - } - } + CodecUtils.close(stream); } /* diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/PTableStatsImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/PTableStatsImpl.java index c6b8d921918..81c73feabd0 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/PTableStatsImpl.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/PTableStatsImpl.java @@ -21,7 +21,6 @@ import java.io.DataInput; import java.io.DataInputStream; import java.io.EOFException; -import java.io.IOException; import java.util.Map; import java.util.SortedMap; import java.util.TreeMap; @@ -80,12 +79,7 @@ public String toString() { try { if (keys.getLength() != 0) { DataInput input = new DataInputStream(stream); - PrefixByteDecoder decoder; - if (entry.getValue().getMaxLength() > 0) { - decoder = new PrefixByteDecoder(entry.getValue().getMaxLength()); - } else { - decoder = new PrefixByteDecoder(); - } + PrefixByteDecoder decoder = new PrefixByteDecoder(entry.getValue().getMaxLength()); try { while (true) { ImmutableBytesWritable ptr = CodecUtils.decode(decoder, input); @@ -95,19 +89,13 @@ public String toString() { } catch (EOFException e) { // Ignore as this signifies we're done } finally { - try { - stream.close(); - } catch (IOException e) {} + CodecUtils.close(stream); } buf.setLength(buf.length() - 1); } buf.append(")"); } finally { - if (stream != null) { - try { - stream.close(); - } catch (IOException e) {} - } + CodecUtils.close(stream); } } buf.append("]"); diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsWriter.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsWriter.java index 1dcfb3df265..188434ac356 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsWriter.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsWriter.java @@ -49,6 +49,7 @@ import org.apache.phoenix.schema.types.PDate; import org.apache.phoenix.schema.types.PLong; import org.apache.phoenix.util.ByteUtil; +import org.apache.phoenix.util.CodecUtils; import org.apache.phoenix.util.PrefixByteDecoder; import org.apache.phoenix.util.ServerUtil; import org.apache.phoenix.util.TimeKeeper; @@ -143,7 +144,7 @@ public void addStats(StatisticsCollector tracker, ImmutableBytesPtr cfKey, List< ImmutableBytesWritable keys = gps.getGuidePosts(); ByteArrayInputStream stream = new ByteArrayInputStream(keys.get(), keys.getOffset(), keys.getLength()); DataInput input = new DataInputStream(stream); - PrefixByteDecoder decoder = new PrefixByteDecoder(); + PrefixByteDecoder decoder = new PrefixByteDecoder(gps.getMaxLength()); try { while (true) { ImmutableBytesWritable ptr = decoder.decode(input); @@ -165,11 +166,7 @@ public void addStats(StatisticsCollector tracker, ImmutableBytesPtr cfKey, List< } catch (EOFException e) { // Ignore as this signifies we're done } finally { - try { - if (stream != null) { - stream.close(); - } - } catch (IOException e) {} + CodecUtils.close(stream); } } } diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/CodecUtils.java b/phoenix-core/src/main/java/org/apache/phoenix/util/CodecUtils.java index cc160bb53a0..f4b330269e9 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/util/CodecUtils.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/util/CodecUtils.java @@ -18,6 +18,7 @@ package org.apache.phoenix.util; import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; import java.io.DataInput; import java.io.DataInputStream; import java.io.DataOutput; @@ -35,12 +36,7 @@ public class CodecUtils { public static List decodeBytes(ImmutableBytesWritable encodedBytes, int maxLength) throws IOException { ByteArrayInputStream stream = new ByteArrayInputStream(encodedBytes.get(), encodedBytes.getOffset(), encodedBytes.getLength()); DataInput input = new DataInputStream(stream); - PrefixByteDecoder decoder; - if (maxLength > 0) { - decoder = new PrefixByteDecoder(maxLength); - } else { - decoder = new PrefixByteDecoder(); - } + PrefixByteDecoder decoder = new PrefixByteDecoder(maxLength); List listOfBytes = Lists.newArrayList(); try { while (true) { @@ -62,7 +58,7 @@ public static int encodeBytes(List listOfBytes, ImmutableBytesWritable p for (byte[] bytes : listOfBytes) { encoder.encode(output, bytes, 0, bytes.length); } - stream.close(); + close(stream); ptr.set(stream.getBuffer(), 0, stream.size()); return encoder.getMaxLength(); } @@ -85,5 +81,24 @@ public static ImmutableBytesWritable decode(PrefixByteDecoder decoder, DataInput throw new RuntimeException(e); } } - + + public static void close(ByteArrayInputStream stream) { + if (stream != null) { + try { + stream.close(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + } + + public static void close(ByteArrayOutputStream stream) { + if (stream != null) { + try { + stream.close(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + } } diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/PrefixByteDecoder.java b/phoenix-core/src/main/java/org/apache/phoenix/util/PrefixByteDecoder.java index 420ac83d9eb..565f0bcaa7a 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/util/PrefixByteDecoder.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/util/PrefixByteDecoder.java @@ -47,8 +47,13 @@ public PrefixByteDecoder() { * @param maxLength maximum length needed for any call to {@link #decode(DataInput)}. */ public PrefixByteDecoder(int maxLength) { - this.maxLength = maxLength; - this.previous = new ImmutableBytesWritable(new byte[maxLength],0,0); + if (maxLength > 0) { + this.maxLength = maxLength; + this.previous = new ImmutableBytesWritable(new byte[maxLength], 0, 0); + } else { + this.maxLength = -1; + previous = new ImmutableBytesWritable(ByteUtil.EMPTY_BYTE_ARRAY); + } } /** diff --git a/phoenix-core/src/test/java/org/apache/phoenix/compile/ScanRangesIntersectTest.java b/phoenix-core/src/test/java/org/apache/phoenix/compile/ScanRangesIntersectTest.java index c2bbc06694f..a3e3e2411d5 100644 --- a/phoenix-core/src/test/java/org/apache/phoenix/compile/ScanRangesIntersectTest.java +++ b/phoenix-core/src/test/java/org/apache/phoenix/compile/ScanRangesIntersectTest.java @@ -27,6 +27,7 @@ import java.util.List; import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import org.apache.phoenix.filter.SkipScanFilter; import org.apache.phoenix.query.KeyRange; import org.apache.phoenix.schema.types.PVarchar; @@ -51,7 +52,8 @@ private static void assertIntersect(ScanRanges ranges, String lowerRange, String scan.setFilter(ranges.getSkipScanFilter()); byte[] startKey = lowerRange == null ? KeyRange.UNBOUND : PVarchar.INSTANCE.toBytes(lowerRange); byte[] stopKey = upperRange == null ? KeyRange.UNBOUND : PVarchar.INSTANCE.toBytes(upperRange); - Scan newScan = ranges.intersectScan(scan, startKey, stopKey, 0, true); + Scan newScan = ranges.intersectScan(scan, new ImmutableBytesWritable(startKey, 0, startKey.length), + new ImmutableBytesWritable(stopKey, 0, stopKey.length), 0, true); if (expectedPoints.length == 0) { assertNull(newScan); } else { From 8161a853a4e35e4a9f27a9f51c1d25bdc6c6810e Mon Sep 17 00:00:00 2001 From: Ankit Singhal Date: Sat, 16 Jan 2016 02:25:20 +0530 Subject: [PATCH 35/47] PHOENIX-2417 Minor nit --- .../org/apache/phoenix/compile/ScanRangesIntersectTest.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/phoenix-core/src/test/java/org/apache/phoenix/compile/ScanRangesIntersectTest.java b/phoenix-core/src/test/java/org/apache/phoenix/compile/ScanRangesIntersectTest.java index a3e3e2411d5..c2bbc06694f 100644 --- a/phoenix-core/src/test/java/org/apache/phoenix/compile/ScanRangesIntersectTest.java +++ b/phoenix-core/src/test/java/org/apache/phoenix/compile/ScanRangesIntersectTest.java @@ -27,7 +27,6 @@ import java.util.List; import org.apache.hadoop.hbase.client.Scan; -import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import org.apache.phoenix.filter.SkipScanFilter; import org.apache.phoenix.query.KeyRange; import org.apache.phoenix.schema.types.PVarchar; @@ -52,8 +51,7 @@ private static void assertIntersect(ScanRanges ranges, String lowerRange, String scan.setFilter(ranges.getSkipScanFilter()); byte[] startKey = lowerRange == null ? KeyRange.UNBOUND : PVarchar.INSTANCE.toBytes(lowerRange); byte[] stopKey = upperRange == null ? KeyRange.UNBOUND : PVarchar.INSTANCE.toBytes(upperRange); - Scan newScan = ranges.intersectScan(scan, new ImmutableBytesWritable(startKey, 0, startKey.length), - new ImmutableBytesWritable(stopKey, 0, stopKey.length), 0, true); + Scan newScan = ranges.intersectScan(scan, startKey, stopKey, 0, true); if (expectedPoints.length == 0) { assertNull(newScan); } else { From f58052f9f71d9f0f80a552946e8ac9f8066916e0 Mon Sep 17 00:00:00 2001 From: Ankit Singhal Date: Sat, 16 Jan 2016 03:17:20 +0530 Subject: [PATCH 36/47] PHOENIX-2417 --- .../org/apache/phoenix/iterate/BaseResultIterators.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java b/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java index e17ce86bf08..8977217bbc1 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java @@ -509,12 +509,13 @@ private List> getParallelScans(byte[] startKey, byte[] stopKey) throw input = new DataInputStream(stream); decoder = new PrefixByteDecoder(gps.getMaxLength()); try { - while (currentKey.compareTo(currentGuidePost = CodecUtils.decode(decoder, input)) >= 0) { + while (currentKey.compareTo(currentGuidePost = CodecUtils.decode(decoder, input)) >= 0 + && currentKey.getLength() != 0) { guideIndex++; } } catch (EOFException e) {} } - byte[] currentKeyBytes = ByteUtil.copyKeyBytesIfNecessary(currentKey); + byte[] currentKeyBytes = currentKey.copyBytes(); // Merge bisect with guideposts for all but the last region while (regionIndex <= stopIndex) { @@ -538,7 +539,7 @@ private List> getParallelScans(byte[] startKey, byte[] stopKey) throw scans = addNewScan(parallelScans, scans, newScan, currentGuidePostBytes, false, regionLocation); currentKeyBytes = currentGuidePost.copyBytes(); currentGuidePost = CodecUtils.decode(decoder, input); - currentGuidePostBytes = ByteUtil.copyKeyBytesIfNecessary(currentGuidePost); + currentGuidePostBytes = currentGuidePost.copyBytes(); guideIndex++; } } catch (EOFException e) {} From fdcb5682e8568829b85d5b939871a6c5f4d26f9e Mon Sep 17 00:00:00 2001 From: Ankit Singhal Date: Sat, 16 Jan 2016 21:35:07 +0530 Subject: [PATCH 37/47] review comments second round --- .../coprocessor/generated/PTableProtos.java | 361 +++++++++++++----- .../phoenix/iterate/BaseResultIterators.java | 5 +- .../phoenix/schema/stats/GuidePostsInfo.java | 76 +--- .../schema/stats/GuidePostsInfoWriter.java | 122 ++++++ .../schema/stats/StatisticsCollector.java | 50 +-- .../phoenix/schema/stats/StatisticsUtil.java | 41 +- phoenix-protocol/src/main/PTable.proto | 5 +- 7 files changed, 450 insertions(+), 210 deletions(-) create mode 100644 phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsInfoWriter.java diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/PTableProtos.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/PTableProtos.java index 026d4def947..28f5d24829a 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/PTableProtos.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/PTableProtos.java @@ -1843,15 +1843,19 @@ public interface PTableStatsOrBuilder */ com.google.protobuf.ByteString getKey(); - // optional bytes guidePosts = 2; + // repeated bytes values = 2; /** - * optional bytes guidePosts = 2; + * repeated bytes values = 2; */ - boolean hasGuidePosts(); + java.util.List getValuesList(); /** - * optional bytes guidePosts = 2; + * repeated bytes values = 2; */ - com.google.protobuf.ByteString getGuidePosts(); + int getValuesCount(); + /** + * repeated bytes values = 2; + */ + com.google.protobuf.ByteString getValues(int index); // optional int64 guidePostsByteCount = 3; /** @@ -1897,13 +1901,23 @@ public interface PTableStatsOrBuilder */ org.apache.phoenix.coprocessor.generated.PGuidePostsProtos.PGuidePostsOrBuilder getPGuidePostsOrBuilder(); - // optional int32 maxLength = 7; + // optional bytes guidePosts = 7; + /** + * optional bytes guidePosts = 7; + */ + boolean hasGuidePosts(); /** - * optional int32 maxLength = 7; + * optional bytes guidePosts = 7; + */ + com.google.protobuf.ByteString getGuidePosts(); + + // optional int32 maxLength = 8; + /** + * optional int32 maxLength = 8; */ boolean hasMaxLength(); /** - * optional int32 maxLength = 7; + * optional int32 maxLength = 8; */ int getMaxLength(); } @@ -1964,28 +1978,31 @@ private PTableStats( break; } case 18: { - bitField0_ |= 0x00000002; - guidePosts_ = input.readBytes(); + if (!((mutable_bitField0_ & 0x00000002) == 0x00000002)) { + values_ = new java.util.ArrayList(); + mutable_bitField0_ |= 0x00000002; + } + values_.add(input.readBytes()); break; } case 24: { - bitField0_ |= 0x00000004; + bitField0_ |= 0x00000002; guidePostsByteCount_ = input.readInt64(); break; } case 32: { - bitField0_ |= 0x00000008; + bitField0_ |= 0x00000004; keyBytesCount_ = input.readInt64(); break; } case 40: { - bitField0_ |= 0x00000010; + bitField0_ |= 0x00000008; guidePostsCount_ = input.readInt32(); break; } case 50: { org.apache.phoenix.coprocessor.generated.PGuidePostsProtos.PGuidePosts.Builder subBuilder = null; - if (((bitField0_ & 0x00000020) == 0x00000020)) { + if (((bitField0_ & 0x00000010) == 0x00000010)) { subBuilder = pGuidePosts_.toBuilder(); } pGuidePosts_ = input.readMessage(org.apache.phoenix.coprocessor.generated.PGuidePostsProtos.PGuidePosts.PARSER, extensionRegistry); @@ -1993,10 +2010,15 @@ private PTableStats( subBuilder.mergeFrom(pGuidePosts_); pGuidePosts_ = subBuilder.buildPartial(); } + bitField0_ |= 0x00000010; + break; + } + case 58: { bitField0_ |= 0x00000020; + guidePosts_ = input.readBytes(); break; } - case 56: { + case 64: { bitField0_ |= 0x00000040; maxLength_ = input.readInt32(); break; @@ -2009,6 +2031,9 @@ private PTableStats( throw new com.google.protobuf.InvalidProtocolBufferException( e.getMessage()).setUnfinishedMessage(this); } finally { + if (((mutable_bitField0_ & 0x00000002) == 0x00000002)) { + values_ = java.util.Collections.unmodifiableList(values_); + } this.unknownFields = unknownFields.build(); makeExtensionsImmutable(); } @@ -2057,20 +2082,27 @@ public com.google.protobuf.ByteString getKey() { return key_; } - // optional bytes guidePosts = 2; - public static final int GUIDEPOSTS_FIELD_NUMBER = 2; - private com.google.protobuf.ByteString guidePosts_; + // repeated bytes values = 2; + public static final int VALUES_FIELD_NUMBER = 2; + private java.util.List values_; /** - * optional bytes guidePosts = 2; + * repeated bytes values = 2; */ - public boolean hasGuidePosts() { - return ((bitField0_ & 0x00000002) == 0x00000002); + public java.util.List + getValuesList() { + return values_; } /** - * optional bytes guidePosts = 2; + * repeated bytes values = 2; */ - public com.google.protobuf.ByteString getGuidePosts() { - return guidePosts_; + public int getValuesCount() { + return values_.size(); + } + /** + * repeated bytes values = 2; + */ + public com.google.protobuf.ByteString getValues(int index) { + return values_.get(index); } // optional int64 guidePostsByteCount = 3; @@ -2080,7 +2112,7 @@ public com.google.protobuf.ByteString getGuidePosts() { * optional int64 guidePostsByteCount = 3; */ public boolean hasGuidePostsByteCount() { - return ((bitField0_ & 0x00000004) == 0x00000004); + return ((bitField0_ & 0x00000002) == 0x00000002); } /** * optional int64 guidePostsByteCount = 3; @@ -2096,7 +2128,7 @@ public long getGuidePostsByteCount() { * optional int64 keyBytesCount = 4; */ public boolean hasKeyBytesCount() { - return ((bitField0_ & 0x00000008) == 0x00000008); + return ((bitField0_ & 0x00000004) == 0x00000004); } /** * optional int64 keyBytesCount = 4; @@ -2112,7 +2144,7 @@ public long getKeyBytesCount() { * optional int32 guidePostsCount = 5; */ public boolean hasGuidePostsCount() { - return ((bitField0_ & 0x00000010) == 0x00000010); + return ((bitField0_ & 0x00000008) == 0x00000008); } /** * optional int32 guidePostsCount = 5; @@ -2128,7 +2160,7 @@ public int getGuidePostsCount() { * optional .PGuidePosts pGuidePosts = 6; */ public boolean hasPGuidePosts() { - return ((bitField0_ & 0x00000020) == 0x00000020); + return ((bitField0_ & 0x00000010) == 0x00000010); } /** * optional .PGuidePosts pGuidePosts = 6; @@ -2143,17 +2175,33 @@ public org.apache.phoenix.coprocessor.generated.PGuidePostsProtos.PGuidePostsOrB return pGuidePosts_; } - // optional int32 maxLength = 7; - public static final int MAXLENGTH_FIELD_NUMBER = 7; + // optional bytes guidePosts = 7; + public static final int GUIDEPOSTS_FIELD_NUMBER = 7; + private com.google.protobuf.ByteString guidePosts_; + /** + * optional bytes guidePosts = 7; + */ + public boolean hasGuidePosts() { + return ((bitField0_ & 0x00000020) == 0x00000020); + } + /** + * optional bytes guidePosts = 7; + */ + public com.google.protobuf.ByteString getGuidePosts() { + return guidePosts_; + } + + // optional int32 maxLength = 8; + public static final int MAXLENGTH_FIELD_NUMBER = 8; private int maxLength_; /** - * optional int32 maxLength = 7; + * optional int32 maxLength = 8; */ public boolean hasMaxLength() { return ((bitField0_ & 0x00000040) == 0x00000040); } /** - * optional int32 maxLength = 7; + * optional int32 maxLength = 8; */ public int getMaxLength() { return maxLength_; @@ -2161,11 +2209,12 @@ public int getMaxLength() { private void initFields() { key_ = com.google.protobuf.ByteString.EMPTY; - guidePosts_ = com.google.protobuf.ByteString.EMPTY; + values_ = java.util.Collections.emptyList(); guidePostsByteCount_ = 0L; keyBytesCount_ = 0L; guidePostsCount_ = 0; pGuidePosts_ = org.apache.phoenix.coprocessor.generated.PGuidePostsProtos.PGuidePosts.getDefaultInstance(); + guidePosts_ = com.google.protobuf.ByteString.EMPTY; maxLength_ = 0; } private byte memoizedIsInitialized = -1; @@ -2187,23 +2236,26 @@ public void writeTo(com.google.protobuf.CodedOutputStream output) if (((bitField0_ & 0x00000001) == 0x00000001)) { output.writeBytes(1, key_); } + for (int i = 0; i < values_.size(); i++) { + output.writeBytes(2, values_.get(i)); + } if (((bitField0_ & 0x00000002) == 0x00000002)) { - output.writeBytes(2, guidePosts_); + output.writeInt64(3, guidePostsByteCount_); } if (((bitField0_ & 0x00000004) == 0x00000004)) { - output.writeInt64(3, guidePostsByteCount_); + output.writeInt64(4, keyBytesCount_); } if (((bitField0_ & 0x00000008) == 0x00000008)) { - output.writeInt64(4, keyBytesCount_); + output.writeInt32(5, guidePostsCount_); } if (((bitField0_ & 0x00000010) == 0x00000010)) { - output.writeInt32(5, guidePostsCount_); + output.writeMessage(6, pGuidePosts_); } if (((bitField0_ & 0x00000020) == 0x00000020)) { - output.writeMessage(6, pGuidePosts_); + output.writeBytes(7, guidePosts_); } if (((bitField0_ & 0x00000040) == 0x00000040)) { - output.writeInt32(7, maxLength_); + output.writeInt32(8, maxLength_); } getUnknownFields().writeTo(output); } @@ -2218,29 +2270,38 @@ public int getSerializedSize() { size += com.google.protobuf.CodedOutputStream .computeBytesSize(1, key_); } + { + int dataSize = 0; + for (int i = 0; i < values_.size(); i++) { + dataSize += com.google.protobuf.CodedOutputStream + .computeBytesSizeNoTag(values_.get(i)); + } + size += dataSize; + size += 1 * getValuesList().size(); + } if (((bitField0_ & 0x00000002) == 0x00000002)) { size += com.google.protobuf.CodedOutputStream - .computeBytesSize(2, guidePosts_); + .computeInt64Size(3, guidePostsByteCount_); } if (((bitField0_ & 0x00000004) == 0x00000004)) { size += com.google.protobuf.CodedOutputStream - .computeInt64Size(3, guidePostsByteCount_); + .computeInt64Size(4, keyBytesCount_); } if (((bitField0_ & 0x00000008) == 0x00000008)) { size += com.google.protobuf.CodedOutputStream - .computeInt64Size(4, keyBytesCount_); + .computeInt32Size(5, guidePostsCount_); } if (((bitField0_ & 0x00000010) == 0x00000010)) { size += com.google.protobuf.CodedOutputStream - .computeInt32Size(5, guidePostsCount_); + .computeMessageSize(6, pGuidePosts_); } if (((bitField0_ & 0x00000020) == 0x00000020)) { size += com.google.protobuf.CodedOutputStream - .computeMessageSize(6, pGuidePosts_); + .computeBytesSize(7, guidePosts_); } if (((bitField0_ & 0x00000040) == 0x00000040)) { size += com.google.protobuf.CodedOutputStream - .computeInt32Size(7, maxLength_); + .computeInt32Size(8, maxLength_); } size += getUnknownFields().getSerializedSize(); memoizedSerializedSize = size; @@ -2270,11 +2331,8 @@ public boolean equals(final java.lang.Object obj) { result = result && getKey() .equals(other.getKey()); } - result = result && (hasGuidePosts() == other.hasGuidePosts()); - if (hasGuidePosts()) { - result = result && getGuidePosts() - .equals(other.getGuidePosts()); - } + result = result && getValuesList() + .equals(other.getValuesList()); result = result && (hasGuidePostsByteCount() == other.hasGuidePostsByteCount()); if (hasGuidePostsByteCount()) { result = result && (getGuidePostsByteCount() @@ -2295,6 +2353,11 @@ public boolean equals(final java.lang.Object obj) { result = result && getPGuidePosts() .equals(other.getPGuidePosts()); } + result = result && (hasGuidePosts() == other.hasGuidePosts()); + if (hasGuidePosts()) { + result = result && getGuidePosts() + .equals(other.getGuidePosts()); + } result = result && (hasMaxLength() == other.hasMaxLength()); if (hasMaxLength()) { result = result && (getMaxLength() @@ -2317,9 +2380,9 @@ public int hashCode() { hash = (37 * hash) + KEY_FIELD_NUMBER; hash = (53 * hash) + getKey().hashCode(); } - if (hasGuidePosts()) { - hash = (37 * hash) + GUIDEPOSTS_FIELD_NUMBER; - hash = (53 * hash) + getGuidePosts().hashCode(); + if (getValuesCount() > 0) { + hash = (37 * hash) + VALUES_FIELD_NUMBER; + hash = (53 * hash) + getValuesList().hashCode(); } if (hasGuidePostsByteCount()) { hash = (37 * hash) + GUIDEPOSTSBYTECOUNT_FIELD_NUMBER; @@ -2337,6 +2400,10 @@ public int hashCode() { hash = (37 * hash) + PGUIDEPOSTS_FIELD_NUMBER; hash = (53 * hash) + getPGuidePosts().hashCode(); } + if (hasGuidePosts()) { + hash = (37 * hash) + GUIDEPOSTS_FIELD_NUMBER; + hash = (53 * hash) + getGuidePosts().hashCode(); + } if (hasMaxLength()) { hash = (37 * hash) + MAXLENGTH_FIELD_NUMBER; hash = (53 * hash) + getMaxLength(); @@ -2453,7 +2520,7 @@ public Builder clear() { super.clear(); key_ = com.google.protobuf.ByteString.EMPTY; bitField0_ = (bitField0_ & ~0x00000001); - guidePosts_ = com.google.protobuf.ByteString.EMPTY; + values_ = java.util.Collections.emptyList(); bitField0_ = (bitField0_ & ~0x00000002); guidePostsByteCount_ = 0L; bitField0_ = (bitField0_ & ~0x00000004); @@ -2467,8 +2534,10 @@ public Builder clear() { pGuidePostsBuilder_.clear(); } bitField0_ = (bitField0_ & ~0x00000020); - maxLength_ = 0; + guidePosts_ = com.google.protobuf.ByteString.EMPTY; bitField0_ = (bitField0_ & ~0x00000040); + maxLength_ = 0; + bitField0_ = (bitField0_ & ~0x00000080); return this; } @@ -2501,24 +2570,25 @@ public org.apache.phoenix.coprocessor.generated.PTableProtos.PTableStats buildPa to_bitField0_ |= 0x00000001; } result.key_ = key_; - if (((from_bitField0_ & 0x00000002) == 0x00000002)) { - to_bitField0_ |= 0x00000002; + if (((bitField0_ & 0x00000002) == 0x00000002)) { + values_ = java.util.Collections.unmodifiableList(values_); + bitField0_ = (bitField0_ & ~0x00000002); } - result.guidePosts_ = guidePosts_; + result.values_ = values_; if (((from_bitField0_ & 0x00000004) == 0x00000004)) { - to_bitField0_ |= 0x00000004; + to_bitField0_ |= 0x00000002; } result.guidePostsByteCount_ = guidePostsByteCount_; if (((from_bitField0_ & 0x00000008) == 0x00000008)) { - to_bitField0_ |= 0x00000008; + to_bitField0_ |= 0x00000004; } result.keyBytesCount_ = keyBytesCount_; if (((from_bitField0_ & 0x00000010) == 0x00000010)) { - to_bitField0_ |= 0x00000010; + to_bitField0_ |= 0x00000008; } result.guidePostsCount_ = guidePostsCount_; if (((from_bitField0_ & 0x00000020) == 0x00000020)) { - to_bitField0_ |= 0x00000020; + to_bitField0_ |= 0x00000010; } if (pGuidePostsBuilder_ == null) { result.pGuidePosts_ = pGuidePosts_; @@ -2526,6 +2596,10 @@ public org.apache.phoenix.coprocessor.generated.PTableProtos.PTableStats buildPa result.pGuidePosts_ = pGuidePostsBuilder_.build(); } if (((from_bitField0_ & 0x00000040) == 0x00000040)) { + to_bitField0_ |= 0x00000020; + } + result.guidePosts_ = guidePosts_; + if (((from_bitField0_ & 0x00000080) == 0x00000080)) { to_bitField0_ |= 0x00000040; } result.maxLength_ = maxLength_; @@ -2548,8 +2622,15 @@ public Builder mergeFrom(org.apache.phoenix.coprocessor.generated.PTableProtos.P if (other.hasKey()) { setKey(other.getKey()); } - if (other.hasGuidePosts()) { - setGuidePosts(other.getGuidePosts()); + if (!other.values_.isEmpty()) { + if (values_.isEmpty()) { + values_ = other.values_; + bitField0_ = (bitField0_ & ~0x00000002); + } else { + ensureValuesIsMutable(); + values_.addAll(other.values_); + } + onChanged(); } if (other.hasGuidePostsByteCount()) { setGuidePostsByteCount(other.getGuidePostsByteCount()); @@ -2563,6 +2644,9 @@ public Builder mergeFrom(org.apache.phoenix.coprocessor.generated.PTableProtos.P if (other.hasPGuidePosts()) { mergePGuidePosts(other.getPGuidePosts()); } + if (other.hasGuidePosts()) { + setGuidePosts(other.getGuidePosts()); + } if (other.hasMaxLength()) { setMaxLength(other.getMaxLength()); } @@ -2633,38 +2717,74 @@ public Builder clearKey() { return this; } - // optional bytes guidePosts = 2; - private com.google.protobuf.ByteString guidePosts_ = com.google.protobuf.ByteString.EMPTY; + // repeated bytes values = 2; + private java.util.List values_ = java.util.Collections.emptyList(); + private void ensureValuesIsMutable() { + if (!((bitField0_ & 0x00000002) == 0x00000002)) { + values_ = new java.util.ArrayList(values_); + bitField0_ |= 0x00000002; + } + } /** - * optional bytes guidePosts = 2; + * repeated bytes values = 2; */ - public boolean hasGuidePosts() { - return ((bitField0_ & 0x00000002) == 0x00000002); + public java.util.List + getValuesList() { + return java.util.Collections.unmodifiableList(values_); } /** - * optional bytes guidePosts = 2; + * repeated bytes values = 2; */ - public com.google.protobuf.ByteString getGuidePosts() { - return guidePosts_; + public int getValuesCount() { + return values_.size(); } /** - * optional bytes guidePosts = 2; + * repeated bytes values = 2; */ - public Builder setGuidePosts(com.google.protobuf.ByteString value) { + public com.google.protobuf.ByteString getValues(int index) { + return values_.get(index); + } + /** + * repeated bytes values = 2; + */ + public Builder setValues( + int index, com.google.protobuf.ByteString value) { if (value == null) { throw new NullPointerException(); } - bitField0_ |= 0x00000002; - guidePosts_ = value; + ensureValuesIsMutable(); + values_.set(index, value); onChanged(); return this; } /** - * optional bytes guidePosts = 2; + * repeated bytes values = 2; */ - public Builder clearGuidePosts() { + public Builder addValues(com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + ensureValuesIsMutable(); + values_.add(value); + onChanged(); + return this; + } + /** + * repeated bytes values = 2; + */ + public Builder addAllValues( + java.lang.Iterable values) { + ensureValuesIsMutable(); + super.addAll(values, values_); + onChanged(); + return this; + } + /** + * repeated bytes values = 2; + */ + public Builder clearValues() { + values_ = java.util.Collections.emptyList(); bitField0_ = (bitField0_ & ~0x00000002); - guidePosts_ = getDefaultInstance().getGuidePosts(); onChanged(); return this; } @@ -2885,34 +3005,70 @@ public org.apache.phoenix.coprocessor.generated.PGuidePostsProtos.PGuidePostsOrB return pGuidePostsBuilder_; } - // optional int32 maxLength = 7; + // optional bytes guidePosts = 7; + private com.google.protobuf.ByteString guidePosts_ = com.google.protobuf.ByteString.EMPTY; + /** + * optional bytes guidePosts = 7; + */ + public boolean hasGuidePosts() { + return ((bitField0_ & 0x00000040) == 0x00000040); + } + /** + * optional bytes guidePosts = 7; + */ + public com.google.protobuf.ByteString getGuidePosts() { + return guidePosts_; + } + /** + * optional bytes guidePosts = 7; + */ + public Builder setGuidePosts(com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000040; + guidePosts_ = value; + onChanged(); + return this; + } + /** + * optional bytes guidePosts = 7; + */ + public Builder clearGuidePosts() { + bitField0_ = (bitField0_ & ~0x00000040); + guidePosts_ = getDefaultInstance().getGuidePosts(); + onChanged(); + return this; + } + + // optional int32 maxLength = 8; private int maxLength_ ; /** - * optional int32 maxLength = 7; + * optional int32 maxLength = 8; */ public boolean hasMaxLength() { - return ((bitField0_ & 0x00000040) == 0x00000040); + return ((bitField0_ & 0x00000080) == 0x00000080); } /** - * optional int32 maxLength = 7; + * optional int32 maxLength = 8; */ public int getMaxLength() { return maxLength_; } /** - * optional int32 maxLength = 7; + * optional int32 maxLength = 8; */ public Builder setMaxLength(int value) { - bitField0_ |= 0x00000040; + bitField0_ |= 0x00000080; maxLength_ = value; onChanged(); return this; } /** - * optional int32 maxLength = 7; + * optional int32 maxLength = 8; */ public Builder clearMaxLength() { - bitField0_ = (bitField0_ & ~0x00000040); + bitField0_ = (bitField0_ & ~0x00000080); maxLength_ = 0; onChanged(); return this; @@ -6943,6 +7099,7 @@ public Builder clearUpdateCacheFrequency() { "\006 \002(\010\022\020\n\010position\030\007 \002(\005\022\021\n\tsortOrder\030\010 \002" + "(\005\022\021\n\tarraySize\030\t \001(\005\022\024\n\014viewConstant\030\n " + "\001(\014\022\026\n\016viewReferenced\030\013 \001(\010\022\022\n\nexpressio" + +<<<<<<< 056ddbed5ca75018213b02d599c7c7a93e8a6692 <<<<<<< 3520e12858223dfcad343e0a3a29c71fe4d074bc "n\030\014 \001(\t\022\026\n\016isRowTimestamp\030\r \001(\010\"\232\001\n\013PTab" + "leStats\022\013\n\003key\030\001 \002(\014\022\016\n\006values\030\002 \003(\014\022\033\n\023" + @@ -6996,6 +7153,34 @@ public Builder clearUpdateCacheFrequency() { "\004JOIN\020\004B@\n(org.apache.phoenix.coprocesso" + "r.generatedB\014PTableProtosH\001\210\001\001\240\001\001" >>>>>>> PHOENIX-2417 Compress memory used by row key byte[] of guideposts +======= + "n\030\014 \001(\t\022\026\n\016isRowTimestamp\030\r \001(\010\"\301\001\n\013PTab" + + "leStats\022\013\n\003key\030\001 \002(\014\022\016\n\006values\030\002 \003(\014\022\033\n\023" + + "guidePostsByteCount\030\003 \001(\003\022\025\n\rkeyBytesCou", + "nt\030\004 \001(\003\022\027\n\017guidePostsCount\030\005 \001(\005\022!\n\013pGu" + + "idePosts\030\006 \001(\0132\014.PGuidePosts\022\022\n\nguidePos" + + "ts\030\007 \001(\014\022\021\n\tmaxLength\030\010 \001(\005\"\206\005\n\006PTable\022\027" + + "\n\017schemaNameBytes\030\001 \002(\014\022\026\n\016tableNameByte" + + "s\030\002 \002(\014\022\036\n\ttableType\030\003 \002(\0162\013.PTableType\022" + + "\022\n\nindexState\030\004 \001(\t\022\026\n\016sequenceNumber\030\005 " + + "\002(\003\022\021\n\ttimeStamp\030\006 \002(\003\022\023\n\013pkNameBytes\030\007 " + + "\001(\014\022\021\n\tbucketNum\030\010 \002(\005\022\031\n\007columns\030\t \003(\0132" + + "\010.PColumn\022\030\n\007indexes\030\n \003(\0132\007.PTable\022\027\n\017i" + + "sImmutableRows\030\013 \002(\010\022 \n\nguidePosts\030\014 \003(\013", + "2\014.PTableStats\022\032\n\022dataTableNameBytes\030\r \001" + + "(\014\022\031\n\021defaultFamilyName\030\016 \001(\014\022\022\n\ndisable" + + "WAL\030\017 \002(\010\022\023\n\013multiTenant\030\020 \002(\010\022\020\n\010viewTy" + + "pe\030\021 \001(\014\022\025\n\rviewStatement\030\022 \001(\014\022\025\n\rphysi" + + "calNames\030\023 \003(\014\022\020\n\010tenantId\030\024 \001(\014\022\023\n\013view" + + "IndexId\030\025 \001(\005\022\021\n\tindexType\030\026 \001(\014\022\026\n\016stat" + + "sTimeStamp\030\027 \001(\003\022\022\n\nstoreNulls\030\030 \001(\010\022\027\n\017" + + "baseColumnCount\030\031 \001(\005\022\036\n\026rowKeyOrderOpti" + + "mizable\030\032 \001(\010\022\025\n\rtransactional\030\033 \001(\010*A\n\n" + + "PTableType\022\n\n\006SYSTEM\020\000\022\010\n\004USER\020\001\022\010\n\004VIEW", + "\020\002\022\t\n\005INDEX\020\003\022\010\n\004JOIN\020\004B@\n(org.apache.ph" + + "oenix.coprocessor.generatedB\014PTableProto" + + "sH\001\210\001\001\240\001\001" +>>>>>>> review comments second round }; com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner = new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() { @@ -7013,7 +7198,7 @@ public com.google.protobuf.ExtensionRegistry assignDescriptors( internal_static_PTableStats_fieldAccessorTable = new com.google.protobuf.GeneratedMessage.FieldAccessorTable( internal_static_PTableStats_descriptor, - new java.lang.String[] { "Key", "GuidePosts", "GuidePostsByteCount", "KeyBytesCount", "GuidePostsCount", "PGuidePosts", "MaxLength", }); + new java.lang.String[] { "Key", "Values", "GuidePostsByteCount", "KeyBytesCount", "GuidePostsCount", "PGuidePosts", "GuidePosts", "MaxLength", }); internal_static_PTable_descriptor = getDescriptor().getMessageTypes().get(2); internal_static_PTable_fieldAccessorTable = new diff --git a/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java b/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java index 8977217bbc1..d1300b54429 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java @@ -464,9 +464,6 @@ private List> getParallelScans(byte[] startKey, byte[] stopKey) throw whereConditions.add(where.getFirst()); } GuidePostsInfo gps = getGuidePosts(whereConditions); - if (logger.isDebugEnabled()) { - logger.debug("Guideposts: " + gps); - } boolean traverseAllRegions = isSalted || isLocalIndex; if (!traverseAllRegions) { byte[] scanStartRow = scan.getStartRow(); @@ -492,7 +489,7 @@ private List> getParallelScans(byte[] startKey, byte[] stopKey) throw } List> parallelScans = Lists.newArrayListWithExpectedSize(stopIndex - regionIndex + 1); - ImmutableBytesWritable currentKey = new ImmutableBytesWritable(startKey, 0, startKey.length); + ImmutableBytesWritable currentKey = new ImmutableBytesWritable(startKey); int gpsSize = gps.getGuidePostsCount(); int estGuidepostsPerRegion = gpsSize == 0 ? 1 : gpsSize / regionLocations.size() + 1; diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsInfo.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsInfo.java index b7f5c49f87c..da7d3a51358 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsInfo.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsInfo.java @@ -17,15 +17,8 @@ */ package org.apache.phoenix.schema.stats; -import java.io.DataOutputStream; -import java.io.IOException; - import org.apache.hadoop.hbase.io.ImmutableBytesWritable; -import org.apache.hadoop.hbase.util.Bytes; import org.apache.phoenix.util.ByteUtil; -import org.apache.phoenix.util.CodecUtils; -import org.apache.phoenix.util.PrefixByteEncoder; -import org.apache.phoenix.util.TrustedByteArrayOutputStream; /** * A class that holds the guidePosts of a region and also allows combining the * guidePosts of different regions when the GuidePostsInfo is formed for a table. @@ -57,13 +50,8 @@ public class GuidePostsInfo { public int getMaxLength() { return maxLength; } - - private TrustedByteArrayOutputStream stream; - private PrefixByteEncoder encoder; - private DataOutputStream output; - private byte[] lastRow; + private int guidePostsCount; - private boolean isStreamInitialized; /** * Constructor that creates GuidePostsInfo per region @@ -77,7 +65,6 @@ public GuidePostsInfo(long byteCount, ImmutableBytesWritable guidePosts, long ro this.byteCount = byteCount; this.rowCount = rowCount; this.guidePostsCount = guidePostsCount; - lastRow = ByteUtil.EMPTY_BYTE_ARRAY; } @@ -93,69 +80,8 @@ public long getRowCount() { return this.rowCount; } - public void incrementRowCount() { - this.rowCount++; - } - public int getGuidePostsCount() { return guidePostsCount; } - /** - * The guide posts, rowCount and byteCount are accumulated every time a guidePosts depth is - * reached while collecting stats. - * @param row - * @param byteCount - * @return - * @throws IOException - */ - public boolean encodeAndCollectGuidePost(byte[] row, long byteCount, long rowCount) { - if (row.length != 0 && Bytes.compareTo(lastRow, row) < 0) { - try { - if(!isStreamInitialized){ - stream = new TrustedByteArrayOutputStream(guidePosts.getLength()); - output = new DataOutputStream(stream); - stream.write(ByteUtil.copyKeyBytesIfNecessary(guidePosts)); - encoder = new PrefixByteEncoder(); - isStreamInitialized=true; - } - encoder.encode(output, row, 0, row.length); - this.byteCount += byteCount; - this.guidePostsCount++; - this.maxLength = encoder.getMaxLength(); - this.rowCount += rowCount; - lastRow = row; - return true; - } catch (IOException e) { - return false; - } - } - return false; - } - - public boolean encodeAndCollectGuidePost(byte[] row){ - return encodeAndCollectGuidePost(row, 0, 0); - } - - public boolean encodeAndCollectGuidePost(byte[] row, long byteCount){ - return encodeAndCollectGuidePost(row, byteCount, 0); - } - - public void close() { - CodecUtils.close(stream); - } - - /* - * This needs to be collect once all stats are encoded and stream buffer needs to be copied for retrieval before close() - */ - public void updateGuidePosts() { - if (stream != null) { - this.guidePosts.set(Bytes.copy(stream.getBuffer(), 0, stream.size())); - } - } - - @Override - public String toString() { - return Bytes.toString(this.guidePosts.get()); - } } \ No newline at end of file diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsInfoWriter.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsInfoWriter.java new file mode 100644 index 00000000000..f0963e0a970 --- /dev/null +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsInfoWriter.java @@ -0,0 +1,122 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.schema.stats; + +import java.io.DataOutputStream; +import java.io.IOException; + +import org.apache.hadoop.hbase.io.ImmutableBytesWritable; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.phoenix.util.ByteUtil; +import org.apache.phoenix.util.CodecUtils; +import org.apache.phoenix.util.PrefixByteEncoder; +import org.apache.phoenix.util.TrustedByteArrayOutputStream; + +/* + * Writer to help in writing guidePosts and creating guidePostInfo. This is used when we are collecting stats or reading stats for a table. + * updateGuidePosts() is required to be invoked before calling getGuidePostInfo() in order to get the updated guidePosts + */ + +public class GuidePostsInfoWriter { + private PrefixByteEncoder encoder; + private byte[] lastRow; + private GuidePostsInfo guidePostInfo; + private long byteCount = 0; + private int guidePostsCount; + + /** + * The rowCount that is flattened across the total number of guide posts. + */ + private long rowCount = 0; + + /** + * Maximum length of a guidePost collected + */ + private int maxLength; + private DataOutputStream output; + private TrustedByteArrayOutputStream stream; + private ImmutableBytesWritable guidePosts=new ImmutableBytesWritable(); + + public final static GuidePostsInfo EMPTY_GUIDEPOST = new GuidePostsInfo(0, + new ImmutableBytesWritable(ByteUtil.EMPTY_BYTE_ARRAY), 0, 0, 0); + + public int getMaxLength() { + return maxLength; + } + public GuidePostsInfoWriter(){ + this.stream = new TrustedByteArrayOutputStream(1); + this.output = new DataOutputStream(stream); + this.encoder=new PrefixByteEncoder(); + lastRow = ByteUtil.EMPTY_BYTE_ARRAY; + } + + /** + * The guide posts, rowCount and byteCount are accumulated every time a guidePosts depth is + * reached while collecting stats. + * @param row + * @param byteCount + * @return + * @throws IOException + */ + public boolean writeGuidePosts( byte[] row, long byteCount, long rowCount) { + if (row.length != 0 && Bytes.compareTo(lastRow, row) < 0) { + try { + encoder.encode(output, row, 0, row.length); + this.byteCount += byteCount; + this.guidePostsCount++; + this.maxLength = encoder.getMaxLength(); + this.rowCount += rowCount; + lastRow = row; + return true; + } catch (IOException e) { + return false; + } + } + return false; + } + + public boolean writeGuidePosts(byte[] row){ + return writeGuidePosts(row, 0, 0); + } + + public boolean writeGuidePosts(byte[] row, long byteCount){ + return writeGuidePosts(row, byteCount, 0); + } + + /* + * This needs to be called once all stats are encoded and stream buffer needs to be copied for retrieval before + * close() + */ + public void updateGuidePosts() { + this.guidePosts.set(Bytes.copy(stream.getBuffer(), 0, stream.size())); + this.guidePostInfo = new GuidePostsInfo(this.byteCount, this.guidePosts, this.rowCount, this.maxLength, + this.guidePostsCount); + } + + public GuidePostsInfo getGuidePostInfo() { + return this.guidePostInfo; + } + + public void close() { + CodecUtils.close(stream); + } + + public void incrementRowCount() { + this.rowCount++; + } +} diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollector.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollector.java index e51e2bdca4d..0c619640541 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollector.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollector.java @@ -60,9 +60,9 @@ public class StatisticsCollector { private long guidepostDepth; private long maxTimeStamp = MetaDataProtocol.MIN_TABLE_TIMESTAMP; - private Map> guidePostsMap = Maps.newHashMap(); + private Map> guidePostsInfoWriterMap = Maps.newHashMap(); protected StatisticsWriter statsTable; - private Pair cachedGps = null; + private Pair cachedGps = null; public StatisticsCollector(RegionCoprocessorEnvironment env, String tableName, long clientTimeStamp) throws IOException { @@ -98,8 +98,8 @@ public StatisticsCollector(RegionCoprocessorEnvironment env, String tableName, l // in a compaction we know the one family ahead of time if (family != null) { ImmutableBytesPtr cfKey = new ImmutableBytesPtr(family); - cachedGps = new Pair(0l, new GuidePostsInfo(0, new ImmutableBytesWritable(ByteUtil.EMPTY_BYTE_ARRAY), 0l, 0, 0)); - guidePostsMap.put(cfKey, cachedGps); + cachedGps = new Pair(0l, new GuidePostsInfoWriter()); + guidePostsInfoWriterMap.put(cfKey, cachedGps); } } @@ -109,10 +109,10 @@ public long getMaxTimeStamp() { public void close() throws IOException { this.statsTable.close(); - for (ImmutableBytesPtr ptr : guidePostsMap.keySet()) { - GuidePostsInfo guidePostInfo = guidePostsMap.get(ptr).getSecond(); - if (guidePostInfo != null) { - guidePostInfo.close(); + for (ImmutableBytesPtr ptr : guidePostsInfoWriterMap.keySet()) { + GuidePostsInfoWriter guidePostWriter = guidePostsInfoWriterMap.get(ptr).getSecond(); + if (guidePostWriter != null) { + guidePostWriter.close(); } } } @@ -136,7 +136,7 @@ private void writeStatsToStatsTable(final Region region, boolean delete, List mutations) throws IOException { */ public void collectStatistics(final List results) { Map famMap = Maps.newHashMap(); - List rowTracker = null; + List rowTracker = null; if (cachedGps == null) { - rowTracker = new ArrayList(); + rowTracker = new ArrayList(); } for (Cell cell : results) { KeyValue kv = KeyValueUtil.ensureKeyValue(cell); maxTimeStamp = Math.max(maxTimeStamp, kv.getTimestamp()); - Pair gps; + Pair gps; if (cachedGps == null) { ImmutableBytesPtr cfKey = new ImmutableBytesPtr(kv.getFamilyArray(), kv.getFamilyOffset(), kv.getFamilyLength()); - gps = guidePostsMap.get(cfKey); + gps = guidePostsInfoWriterMap.get(cfKey); if (gps == null) { - gps = new Pair(0l, - new GuidePostsInfo(0,new ImmutableBytesWritable(ByteUtil.EMPTY_BYTE_ARRAY), 0l, 0, 0)); - guidePostsMap.put(cfKey, gps); + gps = new Pair(0l, + new GuidePostsInfoWriter()); + guidePostsInfoWriterMap.put(cfKey, gps); } if (famMap.get(cfKey) == null) { famMap.put(cfKey, true); @@ -196,13 +196,13 @@ public void collectStatistics(final List results) { if (byteCount >= guidepostDepth) { byte[] row = ByteUtil.copyKeyBytesIfNecessary( new ImmutableBytesWritable(kv.getRowArray(), kv.getRowOffset(), kv.getRowLength())); - if (gps.getSecond().encodeAndCollectGuidePost(row, byteCount)) { + if (gps.getSecond().writeGuidePosts(row, byteCount)) { gps.setFirst(0l); } } } if (cachedGps == null) { - for (GuidePostsInfo s : rowTracker) { + for (GuidePostsInfoWriter s : rowTracker) { s.incrementRowCount(); } } else { @@ -226,21 +226,21 @@ protected InternalScanner getInternalScanner(RegionCoprocessorEnvironment env, S } public void clear() { - this.guidePostsMap.clear(); + this.guidePostsInfoWriterMap.clear(); maxTimeStamp = MetaDataProtocol.MIN_TABLE_TIMESTAMP; } public GuidePostsInfo getGuidePosts(ImmutableBytesPtr fam) { - Pair pair = guidePostsMap.get(fam); - if (pair != null) { return pair.getSecond(); } + Pair pair = guidePostsInfoWriterMap.get(fam); + if (pair != null) { return pair.getSecond().getGuidePostInfo(); } return null; } public void updateGuidePosts() { - for (ImmutableBytesPtr cf : guidePostsMap.keySet()) { - GuidePostsInfo guidePostInfo = guidePostsMap.get(cf).getSecond(); - if (guidePostInfo != null) { - guidePostInfo.updateGuidePosts(); + for (ImmutableBytesPtr cf : guidePostsInfoWriterMap.keySet()) { + GuidePostsInfoWriter guidePostWriter = guidePostsInfoWriterMap.get(cf).getSecond(); + if (guidePostWriter != null) { + guidePostWriter.updateGuidePosts(); } } } diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsUtil.java index 5ea7462d140..a1c63d71ce3 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsUtil.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsUtil.java @@ -22,6 +22,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.SortedMap; import java.util.TreeMap; import org.apache.hadoop.hbase.Cell; @@ -129,7 +130,7 @@ public static PTableStats readStatistics(HTableInterface statsHTable, byte[] tab s.addColumn(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES, QueryConstants.EMPTY_COLUMN_BYTES); ResultScanner scanner = null; long timeStamp = MetaDataProtocol.MIN_TABLE_TIMESTAMP; - TreeMap guidePostsPerCf = new TreeMap(Bytes.BYTES_COMPARATOR); + TreeMap guidePostsInfoWriterPerCf = new TreeMap(Bytes.BYTES_COMPARATOR); try { scanner = statsHTable.getScanner(s); Result result = null; @@ -171,36 +172,44 @@ public static PTableStats readStatistics(HTableInterface statsHTable, byte[] tab } if (cfName != null) { byte[] newGPStartKey = getGuidePostsInfoFromRowKey(tableNameBytes, cfName, result.getRow()); - GuidePostsInfo guidePosts = guidePostsPerCf.get(cfName); - if (guidePosts == null) { - guidePosts = new GuidePostsInfo(0l, new ImmutableBytesWritable(ByteUtil.EMPTY_BYTE_ARRAY), 0l, - 0, 0); - guidePostsPerCf.put(cfName, guidePosts); + GuidePostsInfoWriter guidePostsInfoWriter = guidePostsInfoWriterPerCf.get(cfName); + if (guidePostsInfoWriter == null) { + guidePostsInfoWriter = new GuidePostsInfoWriter(); + guidePostsInfoWriterPerCf.put(cfName, guidePostsInfoWriter); } - guidePosts.encodeAndCollectGuidePost(newGPStartKey, byteCount, rowCount); + guidePostsInfoWriter.writeGuidePosts(newGPStartKey, byteCount, rowCount); } } - for (byte[] cf : guidePostsPerCf.keySet()) { - GuidePostsInfo guidePostInfo = guidePostsPerCf.get(cf); - if (guidePostInfo != null) { - guidePostInfo.updateGuidePosts(); + for (byte[] cf : guidePostsInfoWriterPerCf.keySet()) { + GuidePostsInfoWriter guidePostsInfoWriter = guidePostsInfoWriterPerCf.get(cf); + if (guidePostsInfoWriter != null) { + guidePostsInfoWriter.updateGuidePosts(); } } } finally { if (scanner != null) { scanner.close(); } - for (byte[] cf : guidePostsPerCf.keySet()) { - GuidePostsInfo guidePostInfo = guidePostsPerCf.get(cf); - if (guidePostInfo != null) { - guidePostInfo.close(); + for (byte[] cf : guidePostsInfoWriterPerCf.keySet()) { + GuidePostsInfoWriter guidePostsInfoWriter = guidePostsInfoWriterPerCf.get(cf); + if (guidePostsInfoWriter != null) { + guidePostsInfoWriter.close(); } } } - if (!guidePostsPerCf.isEmpty()) { return new PTableStatsImpl(guidePostsPerCf, timeStamp); } + if (!guidePostsInfoWriterPerCf.isEmpty()) { return new PTableStatsImpl(getGuidePostsPerCf(guidePostsInfoWriterPerCf), timeStamp); } return PTableStats.EMPTY_STATS; } + private static SortedMap getGuidePostsPerCf( + TreeMap guidePostsWriterPerCf) { + TreeMap guidePostsPerCf = new TreeMap(Bytes.BYTES_COMPARATOR); + for (byte[] key : guidePostsWriterPerCf.keySet()) { + guidePostsPerCf.put(key, guidePostsWriterPerCf.get(key).getGuidePostInfo()); + } + return guidePostsPerCf; + } + public static long getGuidePostDepth(int guidepostPerRegion, long guidepostWidth, HTableDescriptor tableDesc) { if (guidepostPerRegion > 0) { long maxFileSize = HConstants.DEFAULT_MAX_FILE_SIZE; diff --git a/phoenix-protocol/src/main/PTable.proto b/phoenix-protocol/src/main/PTable.proto index d7b407bba6c..cc81d627d73 100644 --- a/phoenix-protocol/src/main/PTable.proto +++ b/phoenix-protocol/src/main/PTable.proto @@ -52,12 +52,13 @@ message PColumn { message PTableStats { required bytes key = 1; - optional bytes guidePosts = 2; + repeated bytes values = 2; optional int64 guidePostsByteCount = 3; optional int64 keyBytesCount = 4; optional int32 guidePostsCount = 5; optional PGuidePosts pGuidePosts = 6; - optional int32 maxLength = 7; + optional bytes guidePosts = 7; + optional int32 maxLength = 8; } message PTable { From bc3e95fe2128d10c27efa2cf9f0394f0cfc1315e Mon Sep 17 00:00:00 2001 From: Ankit Singhal Date: Sat, 16 Jan 2016 21:55:02 +0530 Subject: [PATCH 38/47] Writer modified --- .../UngroupedAggregateRegionObserver.java | 1 - .../schema/stats/GuidePostsInfoWriter.java | 36 ++++++++++--------- .../schema/stats/StatisticsCollector.java | 11 +----- .../schema/stats/StatisticsScanner.java | 1 - .../phoenix/schema/stats/StatisticsUtil.java | 16 ++++----- 5 files changed, 28 insertions(+), 37 deletions(-) diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java index 63869bb2851..f332e60b960 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java @@ -783,7 +783,6 @@ private long collectStatsInternal() throws IOException { } finally { try { if (noErrors && !compactionRunning) { - stats.updateGuidePosts(); stats.updateStatistic(region); logger.info("UPDATE STATISTICS finished successfully for scanner: " + innerScanner + ". Number of rows scanned: " + rowCount diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsInfoWriter.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsInfoWriter.java index f0963e0a970..c954d349a5a 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsInfoWriter.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsInfoWriter.java @@ -29,13 +29,12 @@ /* * Writer to help in writing guidePosts and creating guidePostInfo. This is used when we are collecting stats or reading stats for a table. - * updateGuidePosts() is required to be invoked before calling getGuidePostInfo() in order to get the updated guidePosts */ public class GuidePostsInfoWriter { private PrefixByteEncoder encoder; private byte[] lastRow; - private GuidePostsInfo guidePostInfo; + private ImmutableBytesWritable guidePosts=new ImmutableBytesWritable(ByteUtil.EMPTY_BYTE_ARRAY); private long byteCount = 0; private int guidePostsCount; @@ -50,7 +49,6 @@ public class GuidePostsInfoWriter { private int maxLength; private DataOutputStream output; private TrustedByteArrayOutputStream stream; - private ImmutableBytesWritable guidePosts=new ImmutableBytesWritable(); public final static GuidePostsInfo EMPTY_GUIDEPOST = new GuidePostsInfo(0, new ImmutableBytesWritable(ByteUtil.EMPTY_BYTE_ARRAY), 0, 0, 0); @@ -97,20 +95,6 @@ public boolean writeGuidePosts(byte[] row){ public boolean writeGuidePosts(byte[] row, long byteCount){ return writeGuidePosts(row, byteCount, 0); } - - /* - * This needs to be called once all stats are encoded and stream buffer needs to be copied for retrieval before - * close() - */ - public void updateGuidePosts() { - this.guidePosts.set(Bytes.copy(stream.getBuffer(), 0, stream.size())); - this.guidePostInfo = new GuidePostsInfo(this.byteCount, this.guidePosts, this.rowCount, this.maxLength, - this.guidePostsCount); - } - - public GuidePostsInfo getGuidePostInfo() { - return this.guidePostInfo; - } public void close() { CodecUtils.close(stream); @@ -119,4 +103,22 @@ public void close() { public void incrementRowCount() { this.rowCount++; } + + public long getByteCount() { + return byteCount; + } + + public int getGuidePostsCount() { + return guidePostsCount; + } + + public long getRowCount() { + return rowCount; + } + + public ImmutableBytesWritable getGuidePosts() { + this.guidePosts.set(stream.getBuffer(), 0, stream.size()); + return guidePosts; + } + } diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollector.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollector.java index 0c619640541..18f7f8a817d 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollector.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollector.java @@ -232,17 +232,8 @@ public void clear() { public GuidePostsInfo getGuidePosts(ImmutableBytesPtr fam) { Pair pair = guidePostsInfoWriterMap.get(fam); - if (pair != null) { return pair.getSecond().getGuidePostInfo(); } + if (pair != null) { return StatisticsUtil.getGuidePostsInfoFromWriter(pair.getSecond()); } return null; } - public void updateGuidePosts() { - for (ImmutableBytesPtr cf : guidePostsInfoWriterMap.keySet()) { - GuidePostsInfoWriter guidePostWriter = guidePostsInfoWriterMap.get(cf).getSecond(); - if (guidePostWriter != null) { - guidePostWriter.updateGuidePosts(); - } - } - } - } diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsScanner.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsScanner.java index ffce33674b3..13e9491e029 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsScanner.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsScanner.java @@ -93,7 +93,6 @@ protected void updateStat(final List results) { @Override public void close() throws IOException { boolean async = config.getBoolean(COMMIT_STATS_ASYNC, DEFAULT_COMMIT_STATS_ASYNC); - tracker.updateGuidePosts(); StatisticsCollectionRunTracker collectionTracker = StatisticsCollectionRunTracker.getInstance(config); StatisticsScannerCallable callable = new StatisticsScannerCallable(); if (!async) { diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsUtil.java index a1c63d71ce3..22542c37b9f 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsUtil.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsUtil.java @@ -180,12 +180,8 @@ public static PTableStats readStatistics(HTableInterface statsHTable, byte[] tab guidePostsInfoWriter.writeGuidePosts(newGPStartKey, byteCount, rowCount); } } - for (byte[] cf : guidePostsInfoWriterPerCf.keySet()) { - GuidePostsInfoWriter guidePostsInfoWriter = guidePostsInfoWriterPerCf.get(cf); - if (guidePostsInfoWriter != null) { - guidePostsInfoWriter.updateGuidePosts(); - } - } + if (!guidePostsInfoWriterPerCf.isEmpty()) { return new PTableStatsImpl( + getGuidePostsPerCf(guidePostsInfoWriterPerCf), timeStamp); } } finally { if (scanner != null) { scanner.close(); @@ -197,7 +193,6 @@ public static PTableStats readStatistics(HTableInterface statsHTable, byte[] tab } } } - if (!guidePostsInfoWriterPerCf.isEmpty()) { return new PTableStatsImpl(getGuidePostsPerCf(guidePostsInfoWriterPerCf), timeStamp); } return PTableStats.EMPTY_STATS; } @@ -205,7 +200,7 @@ private static SortedMap getGuidePostsPerCf( TreeMap guidePostsWriterPerCf) { TreeMap guidePostsPerCf = new TreeMap(Bytes.BYTES_COMPARATOR); for (byte[] key : guidePostsWriterPerCf.keySet()) { - guidePostsPerCf.put(key, guidePostsWriterPerCf.get(key).getGuidePostInfo()); + guidePostsPerCf.put(key, StatisticsUtil.getGuidePostsInfoFromWriter(guidePostsWriterPerCf.get(key))); } return guidePostsPerCf; } @@ -231,4 +226,9 @@ public static byte[] getGuidePostsInfoFromRowKey(byte[] tableNameBytes, byte[] f ptr.set(row, gpOffset, row.length - gpOffset); return ByteUtil.copyKeyBytesIfNecessary(ptr); } + + public static GuidePostsInfo getGuidePostsInfoFromWriter(GuidePostsInfoWriter writer) { + return new GuidePostsInfo(writer.getByteCount(), writer.getGuidePosts(), writer.getRowCount(), + writer.getMaxLength(), writer.getGuidePostsCount()); + } } \ No newline at end of file From 4f237ec350c232579e88ed9f90ca062784f06b35 Mon Sep 17 00:00:00 2001 From: Ankit Singhal Date: Sun, 17 Jan 2016 02:42:08 +0530 Subject: [PATCH 39/47] updated writer --- .../apache/phoenix/schema/stats/GuidePostsInfoWriter.java | 5 +++++ .../apache/phoenix/schema/stats/StatisticsCollector.java | 2 +- .../org/apache/phoenix/schema/stats/StatisticsUtil.java | 7 +------ 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsInfoWriter.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsInfoWriter.java index c954d349a5a..a47521f1a9f 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsInfoWriter.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsInfoWriter.java @@ -120,5 +120,10 @@ public ImmutableBytesWritable getGuidePosts() { this.guidePosts.set(stream.getBuffer(), 0, stream.size()); return guidePosts; } + + public GuidePostsInfo createGuidePostInfo(){ + return new GuidePostsInfo(this.getByteCount(), this.getGuidePosts(), this.getRowCount(), + this.getMaxLength(), this.getGuidePostsCount()); + } } diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollector.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollector.java index 18f7f8a817d..30e13bd37c8 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollector.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollector.java @@ -232,7 +232,7 @@ public void clear() { public GuidePostsInfo getGuidePosts(ImmutableBytesPtr fam) { Pair pair = guidePostsInfoWriterMap.get(fam); - if (pair != null) { return StatisticsUtil.getGuidePostsInfoFromWriter(pair.getSecond()); } + if (pair != null) { return pair.getSecond().createGuidePostInfo(); } return null; } diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsUtil.java index 22542c37b9f..41a92c1a675 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsUtil.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsUtil.java @@ -200,7 +200,7 @@ private static SortedMap getGuidePostsPerCf( TreeMap guidePostsWriterPerCf) { TreeMap guidePostsPerCf = new TreeMap(Bytes.BYTES_COMPARATOR); for (byte[] key : guidePostsWriterPerCf.keySet()) { - guidePostsPerCf.put(key, StatisticsUtil.getGuidePostsInfoFromWriter(guidePostsWriterPerCf.get(key))); + guidePostsPerCf.put(key, guidePostsWriterPerCf.get(key).createGuidePostInfo()); } return guidePostsPerCf; } @@ -226,9 +226,4 @@ public static byte[] getGuidePostsInfoFromRowKey(byte[] tableNameBytes, byte[] f ptr.set(row, gpOffset, row.length - gpOffset); return ByteUtil.copyKeyBytesIfNecessary(ptr); } - - public static GuidePostsInfo getGuidePostsInfoFromWriter(GuidePostsInfoWriter writer) { - return new GuidePostsInfo(writer.getByteCount(), writer.getGuidePosts(), writer.getRowCount(), - writer.getMaxLength(), writer.getGuidePostsCount()); - } } \ No newline at end of file From 5de890d0b2d26781efa29511b378c87930647ec1 Mon Sep 17 00:00:00 2001 From: Ankit Singhal Date: Sun, 17 Jan 2016 02:47:27 +0530 Subject: [PATCH 40/47] test case updated with new writer --- .../apache/phoenix/filter/SkipScanBigFilterTest.java | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanBigFilterTest.java b/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanBigFilterTest.java index b30f8f0505d..7c61ace7a0a 100644 --- a/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanBigFilterTest.java +++ b/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanBigFilterTest.java @@ -26,7 +26,6 @@ import java.util.Properties; import java.util.SortedMap; -import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import org.apache.hadoop.hbase.util.Bytes; import org.apache.phoenix.compile.QueryPlan; import org.apache.phoenix.end2end.Shadower; @@ -39,8 +38,8 @@ import org.apache.phoenix.schema.PTableImpl; import org.apache.phoenix.schema.PTableKey; import org.apache.phoenix.schema.stats.GuidePostsInfo; +import org.apache.phoenix.schema.stats.GuidePostsInfoWriter; import org.apache.phoenix.schema.stats.PTableStats; -import org.apache.phoenix.util.ByteUtil; import org.apache.phoenix.util.PropertiesUtil; import org.apache.phoenix.util.ReadOnlyProps; import org.junit.BeforeClass; @@ -648,12 +647,12 @@ private void testIntersect(byte[][] regionBoundaries, byte[][] guidePosts) throw stmt.execute(); final PTable table = conn.unwrap(PhoenixConnection.class).getTable(new PTableKey(null, "PERF.BIG_OLAP_DOC")); - GuidePostsInfo info = new GuidePostsInfo(0, new ImmutableBytesWritable(ByteUtil.EMPTY_BYTE_ARRAY), 0l, 0, 0); + GuidePostsInfoWriter gpWriter = new GuidePostsInfoWriter(); for (byte[] gp : guidePosts) { - info.encodeAndCollectGuidePost(gp, 1000); + gpWriter.writeGuidePosts(gp, 1000); } - info.updateGuidePosts(); - info.close(); + GuidePostsInfo info = gpWriter.createGuidePostInfo(); + gpWriter.close(); final SortedMap gpMap = Maps.newTreeMap(Bytes.BYTES_COMPARATOR); gpMap.put(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES, info); PTable tableWithStats = PTableImpl.makePTable(table, new PTableStats() { From f7be475842d0b4fc540b0b51c525ec22f67d15f1 Mon Sep 17 00:00:00 2001 From: Ankit Singhal Date: Sun, 17 Jan 2016 14:00:12 +0530 Subject: [PATCH 41/47] Converted GuidePostsInfoWritr to GuidePostsInfoBuilder --- .../{GuidePostsInfoWriter.java => GuidePostsInfoBuilder.java} | 0 .../apache/phoenix/util/{CodecUtils.java => PrefixByteCodec.java} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename phoenix-core/src/main/java/org/apache/phoenix/schema/stats/{GuidePostsInfoWriter.java => GuidePostsInfoBuilder.java} (100%) rename phoenix-core/src/main/java/org/apache/phoenix/util/{CodecUtils.java => PrefixByteCodec.java} (100%) diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsInfoWriter.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsInfoBuilder.java similarity index 100% rename from phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsInfoWriter.java rename to phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsInfoBuilder.java diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/CodecUtils.java b/phoenix-core/src/main/java/org/apache/phoenix/util/PrefixByteCodec.java similarity index 100% rename from phoenix-core/src/main/java/org/apache/phoenix/util/CodecUtils.java rename to phoenix-core/src/main/java/org/apache/phoenix/util/PrefixByteCodec.java From 825842b68b589581751141879db5c4957f0aac50 Mon Sep 17 00:00:00 2001 From: Ankit Singhal Date: Sun, 17 Jan 2016 14:00:25 +0530 Subject: [PATCH 42/47] Converted GuidePostsInfoWritr to GuidePostsInfoBuilder --- .../phoenix/iterate/BaseResultIterators.java | 8 ++-- .../schema/stats/GuidePostsInfoBuilder.java | 48 +++++++------------ .../phoenix/schema/stats/PTableStatsImpl.java | 8 ++-- .../schema/stats/StatisticsCollector.java | 30 +++++------- .../phoenix/schema/stats/StatisticsUtil.java | 18 +++---- .../schema/stats/StatisticsWriter.java | 4 +- .../apache/phoenix/util/PrefixByteCodec.java | 2 +- .../phoenix/filter/SkipScanBigFilterTest.java | 9 ++-- .../util/PrefixByteEncoderDecoderTest.java | 10 ++-- 9 files changed, 54 insertions(+), 83 deletions(-) diff --git a/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java b/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java index d1300b54429..b3235e277da 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java @@ -81,7 +81,7 @@ import org.apache.phoenix.schema.stats.GuidePostsInfo; import org.apache.phoenix.schema.stats.PTableStats; import org.apache.phoenix.util.ByteUtil; -import org.apache.phoenix.util.CodecUtils; +import org.apache.phoenix.util.PrefixByteCodec; import org.apache.phoenix.util.LogUtil; import org.apache.phoenix.util.PrefixByteDecoder; import org.apache.phoenix.util.SQLCloseables; @@ -506,7 +506,7 @@ private List> getParallelScans(byte[] startKey, byte[] stopKey) throw input = new DataInputStream(stream); decoder = new PrefixByteDecoder(gps.getMaxLength()); try { - while (currentKey.compareTo(currentGuidePost = CodecUtils.decode(decoder, input)) >= 0 + while (currentKey.compareTo(currentGuidePost = PrefixByteCodec.decode(decoder, input)) >= 0 && currentKey.getLength() != 0) { guideIndex++; } @@ -535,7 +535,7 @@ private List> getParallelScans(byte[] startKey, byte[] stopKey) throw false); scans = addNewScan(parallelScans, scans, newScan, currentGuidePostBytes, false, regionLocation); currentKeyBytes = currentGuidePost.copyBytes(); - currentGuidePost = CodecUtils.decode(decoder, input); + currentGuidePost = PrefixByteCodec.decode(decoder, input); currentGuidePostBytes = currentGuidePost.copyBytes(); guideIndex++; } @@ -555,7 +555,7 @@ private List> getParallelScans(byte[] startKey, byte[] stopKey) throw if (!scans.isEmpty()) { // Add any remaining scans parallelScans.add(scans); } - CodecUtils.close(stream); + PrefixByteCodec.close(stream); return parallelScans; } diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsInfoBuilder.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsInfoBuilder.java index a47521f1a9f..11a1faaab7c 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsInfoBuilder.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsInfoBuilder.java @@ -23,7 +23,7 @@ import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import org.apache.hadoop.hbase.util.Bytes; import org.apache.phoenix.util.ByteUtil; -import org.apache.phoenix.util.CodecUtils; +import org.apache.phoenix.util.PrefixByteCodec; import org.apache.phoenix.util.PrefixByteEncoder; import org.apache.phoenix.util.TrustedByteArrayOutputStream; @@ -31,7 +31,7 @@ * Writer to help in writing guidePosts and creating guidePostInfo. This is used when we are collecting stats or reading stats for a table. */ -public class GuidePostsInfoWriter { +public class GuidePostsInfoBuilder { private PrefixByteEncoder encoder; private byte[] lastRow; private ImmutableBytesWritable guidePosts=new ImmutableBytesWritable(ByteUtil.EMPTY_BYTE_ARRAY); @@ -56,7 +56,7 @@ public class GuidePostsInfoWriter { public int getMaxLength() { return maxLength; } - public GuidePostsInfoWriter(){ + public GuidePostsInfoBuilder(){ this.stream = new TrustedByteArrayOutputStream(1); this.output = new DataOutputStream(stream); this.encoder=new PrefixByteEncoder(); @@ -71,7 +71,7 @@ public GuidePostsInfoWriter(){ * @return * @throws IOException */ - public boolean writeGuidePosts( byte[] row, long byteCount, long rowCount) { + public boolean addGuidePosts( byte[] row, long byteCount, long rowCount) { if (row.length != 0 && Bytes.compareTo(lastRow, row) < 0) { try { encoder.encode(output, row, 0, row.length); @@ -88,42 +88,26 @@ public boolean writeGuidePosts( byte[] row, long byteCount, long rowCount) { return false; } - public boolean writeGuidePosts(byte[] row){ - return writeGuidePosts(row, 0, 0); + public boolean addGuidePosts(byte[] row){ + return addGuidePosts(row, 0, 0); } - public boolean writeGuidePosts(byte[] row, long byteCount){ - return writeGuidePosts(row, byteCount, 0); + public boolean addGuidePosts(byte[] row, long byteCount){ + return addGuidePosts(row, byteCount, 0); } - public void close() { - CodecUtils.close(stream); + private void close() { + PrefixByteCodec.close(stream); } - public void incrementRowCount() { - this.rowCount++; - } - - public long getByteCount() { - return byteCount; - } - - public int getGuidePostsCount() { - return guidePostsCount; - } - - public long getRowCount() { - return rowCount; - } - - public ImmutableBytesWritable getGuidePosts() { + public GuidePostsInfo build() { this.guidePosts.set(stream.getBuffer(), 0, stream.size()); - return guidePosts; + GuidePostsInfo guidePostsInfo = new GuidePostsInfo(this.byteCount, this.guidePosts, this.rowCount, this.maxLength, this.guidePostsCount); + this.close(); + return guidePostsInfo; } - - public GuidePostsInfo createGuidePostInfo(){ - return new GuidePostsInfo(this.getByteCount(), this.getGuidePosts(), this.getRowCount(), - this.getMaxLength(), this.getGuidePostsCount()); + public void incrementRowCount() { + this.rowCount++; } } diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/PTableStatsImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/PTableStatsImpl.java index 81c73feabd0..dacc213e093 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/PTableStatsImpl.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/PTableStatsImpl.java @@ -28,7 +28,7 @@ import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import org.apache.hadoop.hbase.util.Bytes; import org.apache.phoenix.coprocessor.MetaDataProtocol; -import org.apache.phoenix.util.CodecUtils; +import org.apache.phoenix.util.PrefixByteCodec; import org.apache.phoenix.util.PrefixByteDecoder; import org.apache.phoenix.util.SizedUtil; @@ -82,20 +82,20 @@ public String toString() { PrefixByteDecoder decoder = new PrefixByteDecoder(entry.getValue().getMaxLength()); try { while (true) { - ImmutableBytesWritable ptr = CodecUtils.decode(decoder, input); + ImmutableBytesWritable ptr = PrefixByteCodec.decode(decoder, input); buf.append(Bytes.toStringBinary(ptr.get())); buf.append(","); } } catch (EOFException e) { // Ignore as this signifies we're done } finally { - CodecUtils.close(stream); + PrefixByteCodec.close(stream); } buf.setLength(buf.length() - 1); } buf.append(")"); } finally { - CodecUtils.close(stream); + PrefixByteCodec.close(stream); } } buf.append("]"); diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollector.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollector.java index 30e13bd37c8..3462f2235ca 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollector.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollector.java @@ -60,9 +60,9 @@ public class StatisticsCollector { private long guidepostDepth; private long maxTimeStamp = MetaDataProtocol.MIN_TABLE_TIMESTAMP; - private Map> guidePostsInfoWriterMap = Maps.newHashMap(); + private Map> guidePostsInfoWriterMap = Maps.newHashMap(); protected StatisticsWriter statsTable; - private Pair cachedGps = null; + private Pair cachedGps = null; public StatisticsCollector(RegionCoprocessorEnvironment env, String tableName, long clientTimeStamp) throws IOException { @@ -98,7 +98,7 @@ public StatisticsCollector(RegionCoprocessorEnvironment env, String tableName, l // in a compaction we know the one family ahead of time if (family != null) { ImmutableBytesPtr cfKey = new ImmutableBytesPtr(family); - cachedGps = new Pair(0l, new GuidePostsInfoWriter()); + cachedGps = new Pair(0l, new GuidePostsInfoBuilder()); guidePostsInfoWriterMap.put(cfKey, cachedGps); } } @@ -109,12 +109,6 @@ public long getMaxTimeStamp() { public void close() throws IOException { this.statsTable.close(); - for (ImmutableBytesPtr ptr : guidePostsInfoWriterMap.keySet()) { - GuidePostsInfoWriter guidePostWriter = guidePostsInfoWriterMap.get(ptr).getSecond(); - if (guidePostWriter != null) { - guidePostWriter.close(); - } - } } public void updateStatistic(Region region) { @@ -166,21 +160,21 @@ private void commitStats(List mutations) throws IOException { */ public void collectStatistics(final List results) { Map famMap = Maps.newHashMap(); - List rowTracker = null; + List rowTracker = null; if (cachedGps == null) { - rowTracker = new ArrayList(); + rowTracker = new ArrayList(); } for (Cell cell : results) { KeyValue kv = KeyValueUtil.ensureKeyValue(cell); maxTimeStamp = Math.max(maxTimeStamp, kv.getTimestamp()); - Pair gps; + Pair gps; if (cachedGps == null) { ImmutableBytesPtr cfKey = new ImmutableBytesPtr(kv.getFamilyArray(), kv.getFamilyOffset(), kv.getFamilyLength()); gps = guidePostsInfoWriterMap.get(cfKey); if (gps == null) { - gps = new Pair(0l, - new GuidePostsInfoWriter()); + gps = new Pair(0l, + new GuidePostsInfoBuilder()); guidePostsInfoWriterMap.put(cfKey, gps); } if (famMap.get(cfKey) == null) { @@ -196,13 +190,13 @@ public void collectStatistics(final List results) { if (byteCount >= guidepostDepth) { byte[] row = ByteUtil.copyKeyBytesIfNecessary( new ImmutableBytesWritable(kv.getRowArray(), kv.getRowOffset(), kv.getRowLength())); - if (gps.getSecond().writeGuidePosts(row, byteCount)) { + if (gps.getSecond().addGuidePosts(row, byteCount)) { gps.setFirst(0l); } } } if (cachedGps == null) { - for (GuidePostsInfoWriter s : rowTracker) { + for (GuidePostsInfoBuilder s : rowTracker) { s.incrementRowCount(); } } else { @@ -231,8 +225,8 @@ public void clear() { } public GuidePostsInfo getGuidePosts(ImmutableBytesPtr fam) { - Pair pair = guidePostsInfoWriterMap.get(fam); - if (pair != null) { return pair.getSecond().createGuidePostInfo(); } + Pair pair = guidePostsInfoWriterMap.get(fam); + if (pair != null) { return pair.getSecond().build(); } return null; } diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsUtil.java index 41a92c1a675..5b471041d30 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsUtil.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsUtil.java @@ -130,7 +130,7 @@ public static PTableStats readStatistics(HTableInterface statsHTable, byte[] tab s.addColumn(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES, QueryConstants.EMPTY_COLUMN_BYTES); ResultScanner scanner = null; long timeStamp = MetaDataProtocol.MIN_TABLE_TIMESTAMP; - TreeMap guidePostsInfoWriterPerCf = new TreeMap(Bytes.BYTES_COMPARATOR); + TreeMap guidePostsInfoWriterPerCf = new TreeMap(Bytes.BYTES_COMPARATOR); try { scanner = statsHTable.getScanner(s); Result result = null; @@ -172,12 +172,12 @@ public static PTableStats readStatistics(HTableInterface statsHTable, byte[] tab } if (cfName != null) { byte[] newGPStartKey = getGuidePostsInfoFromRowKey(tableNameBytes, cfName, result.getRow()); - GuidePostsInfoWriter guidePostsInfoWriter = guidePostsInfoWriterPerCf.get(cfName); + GuidePostsInfoBuilder guidePostsInfoWriter = guidePostsInfoWriterPerCf.get(cfName); if (guidePostsInfoWriter == null) { - guidePostsInfoWriter = new GuidePostsInfoWriter(); + guidePostsInfoWriter = new GuidePostsInfoBuilder(); guidePostsInfoWriterPerCf.put(cfName, guidePostsInfoWriter); } - guidePostsInfoWriter.writeGuidePosts(newGPStartKey, byteCount, rowCount); + guidePostsInfoWriter.addGuidePosts(newGPStartKey, byteCount, rowCount); } } if (!guidePostsInfoWriterPerCf.isEmpty()) { return new PTableStatsImpl( @@ -186,21 +186,15 @@ public static PTableStats readStatistics(HTableInterface statsHTable, byte[] tab if (scanner != null) { scanner.close(); } - for (byte[] cf : guidePostsInfoWriterPerCf.keySet()) { - GuidePostsInfoWriter guidePostsInfoWriter = guidePostsInfoWriterPerCf.get(cf); - if (guidePostsInfoWriter != null) { - guidePostsInfoWriter.close(); - } - } } return PTableStats.EMPTY_STATS; } private static SortedMap getGuidePostsPerCf( - TreeMap guidePostsWriterPerCf) { + TreeMap guidePostsWriterPerCf) { TreeMap guidePostsPerCf = new TreeMap(Bytes.BYTES_COMPARATOR); for (byte[] key : guidePostsWriterPerCf.keySet()) { - guidePostsPerCf.put(key, guidePostsWriterPerCf.get(key).createGuidePostInfo()); + guidePostsPerCf.put(key, guidePostsWriterPerCf.get(key).build()); } return guidePostsPerCf; } diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsWriter.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsWriter.java index 188434ac356..d03af7ab0ab 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsWriter.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsWriter.java @@ -49,7 +49,7 @@ import org.apache.phoenix.schema.types.PDate; import org.apache.phoenix.schema.types.PLong; import org.apache.phoenix.util.ByteUtil; -import org.apache.phoenix.util.CodecUtils; +import org.apache.phoenix.util.PrefixByteCodec; import org.apache.phoenix.util.PrefixByteDecoder; import org.apache.phoenix.util.ServerUtil; import org.apache.phoenix.util.TimeKeeper; @@ -166,7 +166,7 @@ public void addStats(StatisticsCollector tracker, ImmutableBytesPtr cfKey, List< } catch (EOFException e) { // Ignore as this signifies we're done } finally { - CodecUtils.close(stream); + PrefixByteCodec.close(stream); } } } diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/PrefixByteCodec.java b/phoenix-core/src/main/java/org/apache/phoenix/util/PrefixByteCodec.java index f4b330269e9..5f4a3441a9b 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/util/PrefixByteCodec.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/util/PrefixByteCodec.java @@ -31,7 +31,7 @@ import com.google.common.collect.Lists; -public class CodecUtils { +public class PrefixByteCodec { public static List decodeBytes(ImmutableBytesWritable encodedBytes, int maxLength) throws IOException { ByteArrayInputStream stream = new ByteArrayInputStream(encodedBytes.get(), encodedBytes.getOffset(), encodedBytes.getLength()); diff --git a/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanBigFilterTest.java b/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanBigFilterTest.java index 7c61ace7a0a..bb5f4084ad8 100644 --- a/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanBigFilterTest.java +++ b/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanBigFilterTest.java @@ -38,7 +38,7 @@ import org.apache.phoenix.schema.PTableImpl; import org.apache.phoenix.schema.PTableKey; import org.apache.phoenix.schema.stats.GuidePostsInfo; -import org.apache.phoenix.schema.stats.GuidePostsInfoWriter; +import org.apache.phoenix.schema.stats.GuidePostsInfoBuilder; import org.apache.phoenix.schema.stats.PTableStats; import org.apache.phoenix.util.PropertiesUtil; import org.apache.phoenix.util.ReadOnlyProps; @@ -647,12 +647,11 @@ private void testIntersect(byte[][] regionBoundaries, byte[][] guidePosts) throw stmt.execute(); final PTable table = conn.unwrap(PhoenixConnection.class).getTable(new PTableKey(null, "PERF.BIG_OLAP_DOC")); - GuidePostsInfoWriter gpWriter = new GuidePostsInfoWriter(); + GuidePostsInfoBuilder gpWriter = new GuidePostsInfoBuilder(); for (byte[] gp : guidePosts) { - gpWriter.writeGuidePosts(gp, 1000); + gpWriter.addGuidePosts(gp, 1000); } - GuidePostsInfo info = gpWriter.createGuidePostInfo(); - gpWriter.close(); + GuidePostsInfo info = gpWriter.build(); final SortedMap gpMap = Maps.newTreeMap(Bytes.BYTES_COMPARATOR); gpMap.put(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES, info); PTable tableWithStats = PTableImpl.makePTable(table, new PTableStats() { diff --git a/phoenix-core/src/test/java/org/apache/phoenix/util/PrefixByteEncoderDecoderTest.java b/phoenix-core/src/test/java/org/apache/phoenix/util/PrefixByteEncoderDecoderTest.java index ecb3efff94c..269ae1fed4a 100644 --- a/phoenix-core/src/test/java/org/apache/phoenix/util/PrefixByteEncoderDecoderTest.java +++ b/phoenix-core/src/test/java/org/apache/phoenix/util/PrefixByteEncoderDecoderTest.java @@ -53,9 +53,9 @@ public class PrefixByteEncoderDecoderTest { public void testEncode() throws IOException { List listOfBytes = Arrays.asList(Bytes.toBytes("aaaaa"), Bytes.toBytes("aaaabb")); ImmutableBytesWritable ptr = new ImmutableBytesWritable(); - int maxLength = CodecUtils.encodeBytes(listOfBytes, ptr); + int maxLength = PrefixByteCodec.encodeBytes(listOfBytes, ptr); assertEquals(6, maxLength); - TrustedByteArrayOutputStream stream = new TrustedByteArrayOutputStream(CodecUtils.calculateSize(listOfBytes)); + TrustedByteArrayOutputStream stream = new TrustedByteArrayOutputStream(PrefixByteCodec.calculateSize(listOfBytes)); DataOutput output = new DataOutputStream(stream); WritableUtils.writeVInt(output, 0); WritableUtils.writeVInt(output, 5); @@ -78,11 +78,11 @@ public void testEncodeDecodeWithNewBuffer() throws IOException { private void testEncodeDecode(boolean useSingleBuffer) throws IOException { ImmutableBytesWritable ptr = new ImmutableBytesWritable(); - int maxLength = CodecUtils.encodeBytes(guideposts, ptr); + int maxLength = PrefixByteCodec.encodeBytes(guideposts, ptr); int encodedSize = ptr.getLength(); - int unencodedSize = CodecUtils.calculateSize(guideposts); + int unencodedSize = PrefixByteCodec.calculateSize(guideposts); assertTrue(encodedSize < unencodedSize); - List listOfBytes = CodecUtils.decodeBytes(ptr, useSingleBuffer ? maxLength : -1); + List listOfBytes = PrefixByteCodec.decodeBytes(ptr, useSingleBuffer ? maxLength : -1); assertListByteArraysEquals(guideposts, listOfBytes); } From 42c808c76d7c92c63a0714ef3c9f6ea3d52ceb00 Mon Sep 17 00:00:00 2001 From: Ankit Singhal Date: Mon, 18 Jan 2016 00:14:41 +0530 Subject: [PATCH 43/47] updated class level comments for GuidePostsInfoBuilder --- .../org/apache/phoenix/schema/stats/GuidePostsInfoBuilder.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsInfoBuilder.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsInfoBuilder.java index 11a1faaab7c..a8244da1ab3 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsInfoBuilder.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsInfoBuilder.java @@ -28,7 +28,7 @@ import org.apache.phoenix.util.TrustedByteArrayOutputStream; /* - * Writer to help in writing guidePosts and creating guidePostInfo. This is used when we are collecting stats or reading stats for a table. + * Builder to help in adding guidePosts and building guidePostInfo. This is used when we are collecting stats or reading stats for a table. */ public class GuidePostsInfoBuilder { From 0852f9eaf62e407b143660b8d2d68f18124cf519 Mon Sep 17 00:00:00 2001 From: Ankit Singhal Date: Wed, 20 Jan 2016 00:34:43 +0530 Subject: [PATCH 44/47] Delete stats rows at server side for 4.7 upgrade --- .../coprocessor/MetaDataRegionObserver.java | 141 +++++++++++++++--- 1 file changed, 122 insertions(+), 19 deletions(-) diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataRegionObserver.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataRegionObserver.java index c40e3cd435c..c133cbef99e 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataRegionObserver.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataRegionObserver.java @@ -18,10 +18,6 @@ package org.apache.phoenix.coprocessor; import java.io.IOException; - -import org.apache.log4j.Level; -import org.apache.log4j.Logger; -import org.apache.phoenix.jdbc.PhoenixDriver; import java.sql.DriverManager; import java.sql.SQLException; import java.util.ArrayList; @@ -36,29 +32,46 @@ import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CoprocessorEnvironment; import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Delete; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.HTableInterface; +import org.apache.hadoop.hbase.client.Mutation; +import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.ResultScanner; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver; import org.apache.hadoop.hbase.coprocessor.ObserverContext; import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment; -import org.apache.hadoop.hbase.filter.SingleColumnValueFilter; import org.apache.hadoop.hbase.filter.CompareFilter; +import org.apache.hadoop.hbase.filter.SingleColumnValueFilter; import org.apache.hadoop.hbase.regionserver.RegionScanner; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.log4j.Level; +import org.apache.log4j.Logger; import org.apache.phoenix.cache.GlobalCache; import org.apache.phoenix.jdbc.PhoenixConnection; import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData; +import org.apache.phoenix.jdbc.PhoenixDriver; +import org.apache.phoenix.query.QueryConstants; import org.apache.phoenix.query.QueryServices; import org.apache.phoenix.query.QueryServicesOptions; import org.apache.phoenix.schema.MetaDataClient; import org.apache.phoenix.schema.PIndexState; -import org.apache.phoenix.schema.types.PLong; import org.apache.phoenix.schema.PTable; import org.apache.phoenix.schema.TableRef; +import org.apache.phoenix.schema.types.PLong; +import org.apache.phoenix.schema.types.PVarchar; +import org.apache.phoenix.util.ByteUtil; +import org.apache.phoenix.util.KeyValueUtil; import org.apache.phoenix.util.MetaDataUtil; import org.apache.phoenix.util.PhoenixRuntime; import org.apache.phoenix.util.SchemaUtil; +import com.google.common.collect.Lists; + /** * Coprocessor for metadata related operations. This coprocessor would only be registered @@ -69,6 +82,8 @@ public class MetaDataRegionObserver extends BaseRegionObserver { protected ScheduledThreadPoolExecutor executor = new ScheduledThreadPoolExecutor(1); private boolean enableRebuildIndex = QueryServicesOptions.DEFAULT_INDEX_FAILURE_HANDLING_REBUILD; private long rebuildIndexTimeInterval = QueryServicesOptions.DEFAULT_INDEX_FAILURE_HANDLING_REBUILD_INTERVAL; + public static final byte[] UPGRADE_TO_4_7_COLUMN_NAME = Bytes.toBytes("UPGRADE_TO_4_7"); + public static final String UPGRADE_TO_4_7_COLUMN_VALUE = "TRUNCATED_4_7"; @Override public void preClose(final ObserverContext c, @@ -96,10 +111,109 @@ public void start(CoprocessorEnvironment env) throws IOException { rebuildIndexTimeInterval = env.getConfiguration().getLong(QueryServices.INDEX_FAILURE_HANDLING_REBUILD_INTERVAL_ATTRIB, QueryServicesOptions.DEFAULT_INDEX_FAILURE_HANDLING_REBUILD_INTERVAL); } - + + private static String getJdbcUrl(RegionCoprocessorEnvironment env) { + String zkQuorum = env.getConfiguration().get(HConstants.ZOOKEEPER_QUORUM); + String zkClientPort = env.getConfiguration().get(HConstants.ZOOKEEPER_CLIENT_PORT, + Integer.toString(HConstants.DEFAULT_ZOOKEPER_CLIENT_PORT)); + String zkParentNode = env.getConfiguration().get(HConstants.ZOOKEEPER_ZNODE_PARENT, + HConstants.DEFAULT_ZOOKEEPER_ZNODE_PARENT); + return PhoenixRuntime.JDBC_PROTOCOL + PhoenixRuntime.JDBC_PROTOCOL_SEPARATOR + zkQuorum + + PhoenixRuntime.JDBC_PROTOCOL_SEPARATOR + zkClientPort + + PhoenixRuntime.JDBC_PROTOCOL_SEPARATOR + zkParentNode; + } @Override public void postOpen(ObserverContext e) { + final RegionCoprocessorEnvironment env = e.getEnvironment(); + + Runnable r = new Runnable() { + @Override + public void run() { + HTableInterface metaTable = null; + HTableInterface statsTable = null; + try { + Thread.sleep(1000); + LOG.info("Stats will be deleted for upgrade 4.7 requirement!!"); + metaTable = env.getTable(TableName.valueOf(PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME)); + List columnCells = metaTable + .get(new Get(SchemaUtil.getTableKey(null, PhoenixDatabaseMetaData.SYSTEM_SCHEMA_NAME, + PhoenixDatabaseMetaData.SYSTEM_CATALOG_TABLE))) + .getColumnCells(PhoenixDatabaseMetaData.TABLE_FAMILY_BYTES, + QueryConstants.EMPTY_COLUMN_BYTES); + if (!columnCells.isEmpty() + && columnCells.get(0).getTimestamp() < MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_7_0) { + + statsTable = env.getTable(TableName.valueOf(PhoenixDatabaseMetaData.SYSTEM_STATS_NAME)); + byte[] statsTableKey = SchemaUtil.getTableKey(null, PhoenixDatabaseMetaData.SYSTEM_SCHEMA_NAME, + PhoenixDatabaseMetaData.SYSTEM_STATS_TABLE); + KeyValue upgradeKV = KeyValueUtil.newKeyValue(statsTableKey, + PhoenixDatabaseMetaData.TABLE_FAMILY_BYTES, UPGRADE_TO_4_7_COLUMN_NAME, + MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_7_0 - 1, + PVarchar.INSTANCE.toBytes(UPGRADE_TO_4_7_COLUMN_VALUE)); + Put upgradePut = new Put(statsTableKey); + upgradePut.add(upgradeKV); + + // check for null in UPGRADE_TO_4_7_COLUMN_NAME in checkAndPut so that only single client + // drop the rows of SYSTEM.STATS + if (metaTable.checkAndPut(statsTableKey, PhoenixDatabaseMetaData.TABLE_FAMILY_BYTES, + UPGRADE_TO_4_7_COLUMN_NAME, null, upgradePut)) { + List mutations = Lists.newArrayListWithExpectedSize(1000); + Scan scan = new Scan(); + scan.setRaw(true); + scan.setMaxVersions(); + ResultScanner statsScanner = statsTable.getScanner(scan); + Result r; + mutations.clear(); + int count = 0; + while ((r = statsScanner.next()) != null) { + Delete delete = null; + for (KeyValue keyValue : r.raw()) { + if (KeyValue.Type.codeToType(keyValue.getType()) == KeyValue.Type.Put) { + if (delete == null) { + delete = new Delete(keyValue.getRow()); + } + KeyValue deleteKeyValue = new KeyValue(keyValue.getRowArray(), + keyValue.getRowOffset(), keyValue.getRowLength(), + keyValue.getFamilyArray(), keyValue.getFamilyOffset(), + keyValue.getFamilyLength(), keyValue.getQualifierArray(), + keyValue.getQualifierOffset(), keyValue.getQualifierLength(), + keyValue.getTimestamp(), KeyValue.Type.Delete, + ByteUtil.EMPTY_BYTE_ARRAY, 0, 0); + delete.addDeleteMarker(deleteKeyValue); + } + } + if (delete != null) { + mutations.add(delete); + if (count > 1000) { + statsTable.batch(mutations); + mutations.clear(); + count = 0; + } + count++; + } + } + if (!mutations.isEmpty()) { + statsTable.batch(mutations); + } + } + } + } catch (Exception e) { + LOG.warn("Exception while deleting stats..", e); + } finally { + try { + if (metaTable != null) { + metaTable.close(); + } + if (statsTable != null) { + statsTable.close(); + } + } catch (IOException e) {} + } + } + }; + (new Thread(r)).start(); + if (!enableRebuildIndex) { LOG.info("Failure Index Rebuild is skipped by configuration."); return; @@ -134,17 +248,6 @@ public BuildIndexScheduleTask(RegionCoprocessorEnvironment env) { this.env = env; } - private String getJdbcUrl() { - String zkQuorum = this.env.getConfiguration().get(HConstants.ZOOKEEPER_QUORUM); - String zkClientPort = this.env.getConfiguration().get(HConstants.ZOOKEEPER_CLIENT_PORT, - Integer.toString(HConstants.DEFAULT_ZOOKEPER_CLIENT_PORT)); - String zkParentNode = this.env.getConfiguration().get(HConstants.ZOOKEEPER_ZNODE_PARENT, - HConstants.DEFAULT_ZOOKEEPER_ZNODE_PARENT); - return PhoenixRuntime.JDBC_PROTOCOL + PhoenixRuntime.JDBC_PROTOCOL_SEPARATOR + zkQuorum - + PhoenixRuntime.JDBC_PROTOCOL_SEPARATOR + zkClientPort - + PhoenixRuntime.JDBC_PROTOCOL_SEPARATOR + zkParentNode; - } - @Override public void run() { RegionScanner scanner = null; @@ -219,7 +322,7 @@ public void run() { } if (conn == null) { - conn = DriverManager.getConnection(getJdbcUrl()).unwrap(PhoenixConnection.class); + conn = DriverManager.getConnection(getJdbcUrl(env)).unwrap(PhoenixConnection.class); } String dataTableFullName = SchemaUtil.getTableName(schemaName, dataTable); From 6f21fbc3dcede77090c3985db90a565f03108d3a Mon Sep 17 00:00:00 2001 From: Ankit Singhal Date: Wed, 20 Jan 2016 01:13:41 +0530 Subject: [PATCH 45/47] Reducing delete batch size from 1000 to 10 for stats table as each row could be too big --- .../org/apache/phoenix/coprocessor/MetaDataRegionObserver.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataRegionObserver.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataRegionObserver.java index c133cbef99e..64d5f3cef34 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataRegionObserver.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataRegionObserver.java @@ -185,7 +185,7 @@ public void run() { } if (delete != null) { mutations.add(delete); - if (count > 1000) { + if (count > 10) { statsTable.batch(mutations); mutations.clear(); count = 0; From f607211fa17baecc730cd8f337e3c731b2841a22 Mon Sep 17 00:00:00 2001 From: Ankit Singhal Date: Wed, 20 Jan 2016 15:15:14 +0530 Subject: [PATCH 46/47] updatd review comments for protobuf backward compatibility and moving code to upgrade Util --- .../coprocessor/MetaDataRegionObserver.java | 86 +--- .../generated/PGuidePostsProtos.java | 371 +++++++++++++----- .../coprocessor/generated/PTableProtos.java | 113 ++++-- .../org/apache/phoenix/schema/PTableImpl.java | 10 +- .../org/apache/phoenix/util/UpgradeUtil.java | 66 ++++ phoenix-protocol/src/main/PGuidePosts.proto | 5 +- phoenix-protocol/src/main/PTable.proto | 2 +- 7 files changed, 418 insertions(+), 235 deletions(-) diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataRegionObserver.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataRegionObserver.java index 64d5f3cef34..7950ac8f183 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataRegionObserver.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataRegionObserver.java @@ -32,15 +32,9 @@ import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CoprocessorEnvironment; import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.client.Delete; -import org.apache.hadoop.hbase.client.Get; import org.apache.hadoop.hbase.client.HTableInterface; -import org.apache.hadoop.hbase.client.Mutation; -import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.Result; -import org.apache.hadoop.hbase.client.ResultScanner; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver; import org.apache.hadoop.hbase.coprocessor.ObserverContext; @@ -55,7 +49,6 @@ import org.apache.phoenix.jdbc.PhoenixConnection; import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData; import org.apache.phoenix.jdbc.PhoenixDriver; -import org.apache.phoenix.query.QueryConstants; import org.apache.phoenix.query.QueryServices; import org.apache.phoenix.query.QueryServicesOptions; import org.apache.phoenix.schema.MetaDataClient; @@ -63,14 +56,10 @@ import org.apache.phoenix.schema.PTable; import org.apache.phoenix.schema.TableRef; import org.apache.phoenix.schema.types.PLong; -import org.apache.phoenix.schema.types.PVarchar; -import org.apache.phoenix.util.ByteUtil; -import org.apache.phoenix.util.KeyValueUtil; import org.apache.phoenix.util.MetaDataUtil; import org.apache.phoenix.util.PhoenixRuntime; import org.apache.phoenix.util.SchemaUtil; - -import com.google.common.collect.Lists; +import org.apache.phoenix.util.UpgradeUtil; /** @@ -82,8 +71,6 @@ public class MetaDataRegionObserver extends BaseRegionObserver { protected ScheduledThreadPoolExecutor executor = new ScheduledThreadPoolExecutor(1); private boolean enableRebuildIndex = QueryServicesOptions.DEFAULT_INDEX_FAILURE_HANDLING_REBUILD; private long rebuildIndexTimeInterval = QueryServicesOptions.DEFAULT_INDEX_FAILURE_HANDLING_REBUILD_INTERVAL; - public static final byte[] UPGRADE_TO_4_7_COLUMN_NAME = Bytes.toBytes("UPGRADE_TO_4_7"); - public static final String UPGRADE_TO_4_7_COLUMN_VALUE = "TRUNCATED_4_7"; @Override public void preClose(final ObserverContext c, @@ -134,72 +121,15 @@ public void run() { HTableInterface statsTable = null; try { Thread.sleep(1000); - LOG.info("Stats will be deleted for upgrade 4.7 requirement!!"); metaTable = env.getTable(TableName.valueOf(PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME)); - List columnCells = metaTable - .get(new Get(SchemaUtil.getTableKey(null, PhoenixDatabaseMetaData.SYSTEM_SCHEMA_NAME, - PhoenixDatabaseMetaData.SYSTEM_CATALOG_TABLE))) - .getColumnCells(PhoenixDatabaseMetaData.TABLE_FAMILY_BYTES, - QueryConstants.EMPTY_COLUMN_BYTES); - if (!columnCells.isEmpty() - && columnCells.get(0).getTimestamp() < MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_7_0) { - - statsTable = env.getTable(TableName.valueOf(PhoenixDatabaseMetaData.SYSTEM_STATS_NAME)); - byte[] statsTableKey = SchemaUtil.getTableKey(null, PhoenixDatabaseMetaData.SYSTEM_SCHEMA_NAME, - PhoenixDatabaseMetaData.SYSTEM_STATS_TABLE); - KeyValue upgradeKV = KeyValueUtil.newKeyValue(statsTableKey, - PhoenixDatabaseMetaData.TABLE_FAMILY_BYTES, UPGRADE_TO_4_7_COLUMN_NAME, - MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_7_0 - 1, - PVarchar.INSTANCE.toBytes(UPGRADE_TO_4_7_COLUMN_VALUE)); - Put upgradePut = new Put(statsTableKey); - upgradePut.add(upgradeKV); - - // check for null in UPGRADE_TO_4_7_COLUMN_NAME in checkAndPut so that only single client - // drop the rows of SYSTEM.STATS - if (metaTable.checkAndPut(statsTableKey, PhoenixDatabaseMetaData.TABLE_FAMILY_BYTES, - UPGRADE_TO_4_7_COLUMN_NAME, null, upgradePut)) { - List mutations = Lists.newArrayListWithExpectedSize(1000); - Scan scan = new Scan(); - scan.setRaw(true); - scan.setMaxVersions(); - ResultScanner statsScanner = statsTable.getScanner(scan); - Result r; - mutations.clear(); - int count = 0; - while ((r = statsScanner.next()) != null) { - Delete delete = null; - for (KeyValue keyValue : r.raw()) { - if (KeyValue.Type.codeToType(keyValue.getType()) == KeyValue.Type.Put) { - if (delete == null) { - delete = new Delete(keyValue.getRow()); - } - KeyValue deleteKeyValue = new KeyValue(keyValue.getRowArray(), - keyValue.getRowOffset(), keyValue.getRowLength(), - keyValue.getFamilyArray(), keyValue.getFamilyOffset(), - keyValue.getFamilyLength(), keyValue.getQualifierArray(), - keyValue.getQualifierOffset(), keyValue.getQualifierLength(), - keyValue.getTimestamp(), KeyValue.Type.Delete, - ByteUtil.EMPTY_BYTE_ARRAY, 0, 0); - delete.addDeleteMarker(deleteKeyValue); - } - } - if (delete != null) { - mutations.add(delete); - if (count > 10) { - statsTable.batch(mutations); - mutations.clear(); - count = 0; - } - count++; - } - } - if (!mutations.isEmpty()) { - statsTable.batch(mutations); - } - } + statsTable = env.getTable(TableName.valueOf(PhoenixDatabaseMetaData.SYSTEM_STATS_NAME)); + if (UpgradeUtil.truncateStats(metaTable, statsTable)) { + LOG.info("Stats are successfully truncated for upgrade 4.7!!"); } - } catch (Exception e) { - LOG.warn("Exception while deleting stats..", e); + } catch (Exception exception) { + LOG.warn("Exception while truncate stats..," + + " please check and delete stats manually inorder to get proper result with old client!!"); + LOG.warn(exception.getStackTrace()); } finally { try { if (metaTable != null) { diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/PGuidePostsProtos.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/PGuidePostsProtos.java index 58999740ed8..d9dcb56c4ab 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/PGuidePostsProtos.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/PGuidePostsProtos.java @@ -11,15 +11,19 @@ public static void registerAllExtensions( public interface PGuidePostsOrBuilder extends com.google.protobuf.MessageOrBuilder { - // optional bytes guidePosts = 1; + // repeated bytes guidePosts = 1; /** - * optional bytes guidePosts = 1; + * repeated bytes guidePosts = 1; */ - boolean hasGuidePosts(); + java.util.List getGuidePostsList(); /** - * optional bytes guidePosts = 1; + * repeated bytes guidePosts = 1; */ - com.google.protobuf.ByteString getGuidePosts(); + int getGuidePostsCount(); + /** + * repeated bytes guidePosts = 1; + */ + com.google.protobuf.ByteString getGuidePosts(int index); // optional int64 byteCount = 2; /** @@ -51,15 +55,25 @@ public interface PGuidePostsOrBuilder */ int getMaxLength(); - // optional int32 guidePostsCount = 5; + // optional int32 encodedGuidePostsCount = 5; /** - * optional int32 guidePostsCount = 5; + * optional int32 encodedGuidePostsCount = 5; */ - boolean hasGuidePostsCount(); + boolean hasEncodedGuidePostsCount(); /** - * optional int32 guidePostsCount = 5; + * optional int32 encodedGuidePostsCount = 5; */ - int getGuidePostsCount(); + int getEncodedGuidePostsCount(); + + // optional bytes encodedGuidePosts = 6; + /** + * optional bytes encodedGuidePosts = 6; + */ + boolean hasEncodedGuidePosts(); + /** + * optional bytes encodedGuidePosts = 6; + */ + com.google.protobuf.ByteString getEncodedGuidePosts(); } /** * Protobuf type {@code PGuidePosts} @@ -113,28 +127,36 @@ private PGuidePosts( break; } case 10: { - bitField0_ |= 0x00000001; - guidePosts_ = input.readBytes(); + if (!((mutable_bitField0_ & 0x00000001) == 0x00000001)) { + guidePosts_ = new java.util.ArrayList(); + mutable_bitField0_ |= 0x00000001; + } + guidePosts_.add(input.readBytes()); break; } case 16: { - bitField0_ |= 0x00000002; + bitField0_ |= 0x00000001; byteCount_ = input.readInt64(); break; } case 24: { - bitField0_ |= 0x00000004; + bitField0_ |= 0x00000002; rowCount_ = input.readInt64(); break; } case 32: { - bitField0_ |= 0x00000008; + bitField0_ |= 0x00000004; maxLength_ = input.readInt32(); break; } case 40: { + bitField0_ |= 0x00000008; + encodedGuidePostsCount_ = input.readInt32(); + break; + } + case 50: { bitField0_ |= 0x00000010; - guidePostsCount_ = input.readInt32(); + encodedGuidePosts_ = input.readBytes(); break; } } @@ -145,6 +167,9 @@ private PGuidePosts( throw new com.google.protobuf.InvalidProtocolBufferException( e.getMessage()).setUnfinishedMessage(this); } finally { + if (((mutable_bitField0_ & 0x00000001) == 0x00000001)) { + guidePosts_ = java.util.Collections.unmodifiableList(guidePosts_); + } this.unknownFields = unknownFields.build(); makeExtensionsImmutable(); } @@ -177,20 +202,27 @@ public com.google.protobuf.Parser getParserForType() { } private int bitField0_; - // optional bytes guidePosts = 1; + // repeated bytes guidePosts = 1; public static final int GUIDEPOSTS_FIELD_NUMBER = 1; - private com.google.protobuf.ByteString guidePosts_; + private java.util.List guidePosts_; /** - * optional bytes guidePosts = 1; + * repeated bytes guidePosts = 1; */ - public boolean hasGuidePosts() { - return ((bitField0_ & 0x00000001) == 0x00000001); + public java.util.List + getGuidePostsList() { + return guidePosts_; } /** - * optional bytes guidePosts = 1; + * repeated bytes guidePosts = 1; */ - public com.google.protobuf.ByteString getGuidePosts() { - return guidePosts_; + public int getGuidePostsCount() { + return guidePosts_.size(); + } + /** + * repeated bytes guidePosts = 1; + */ + public com.google.protobuf.ByteString getGuidePosts(int index) { + return guidePosts_.get(index); } // optional int64 byteCount = 2; @@ -200,7 +232,7 @@ public com.google.protobuf.ByteString getGuidePosts() { * optional int64 byteCount = 2; */ public boolean hasByteCount() { - return ((bitField0_ & 0x00000002) == 0x00000002); + return ((bitField0_ & 0x00000001) == 0x00000001); } /** * optional int64 byteCount = 2; @@ -216,7 +248,7 @@ public long getByteCount() { * optional int64 rowCount = 3; */ public boolean hasRowCount() { - return ((bitField0_ & 0x00000004) == 0x00000004); + return ((bitField0_ & 0x00000002) == 0x00000002); } /** * optional int64 rowCount = 3; @@ -232,7 +264,7 @@ public long getRowCount() { * optional int32 maxLength = 4; */ public boolean hasMaxLength() { - return ((bitField0_ & 0x00000008) == 0x00000008); + return ((bitField0_ & 0x00000004) == 0x00000004); } /** * optional int32 maxLength = 4; @@ -241,28 +273,45 @@ public int getMaxLength() { return maxLength_; } - // optional int32 guidePostsCount = 5; - public static final int GUIDEPOSTSCOUNT_FIELD_NUMBER = 5; - private int guidePostsCount_; + // optional int32 encodedGuidePostsCount = 5; + public static final int ENCODEDGUIDEPOSTSCOUNT_FIELD_NUMBER = 5; + private int encodedGuidePostsCount_; + /** + * optional int32 encodedGuidePostsCount = 5; + */ + public boolean hasEncodedGuidePostsCount() { + return ((bitField0_ & 0x00000008) == 0x00000008); + } + /** + * optional int32 encodedGuidePostsCount = 5; + */ + public int getEncodedGuidePostsCount() { + return encodedGuidePostsCount_; + } + + // optional bytes encodedGuidePosts = 6; + public static final int ENCODEDGUIDEPOSTS_FIELD_NUMBER = 6; + private com.google.protobuf.ByteString encodedGuidePosts_; /** - * optional int32 guidePostsCount = 5; + * optional bytes encodedGuidePosts = 6; */ - public boolean hasGuidePostsCount() { + public boolean hasEncodedGuidePosts() { return ((bitField0_ & 0x00000010) == 0x00000010); } /** - * optional int32 guidePostsCount = 5; + * optional bytes encodedGuidePosts = 6; */ - public int getGuidePostsCount() { - return guidePostsCount_; + public com.google.protobuf.ByteString getEncodedGuidePosts() { + return encodedGuidePosts_; } private void initFields() { - guidePosts_ = com.google.protobuf.ByteString.EMPTY; + guidePosts_ = java.util.Collections.emptyList(); byteCount_ = 0L; rowCount_ = 0L; maxLength_ = 0; - guidePostsCount_ = 0; + encodedGuidePostsCount_ = 0; + encodedGuidePosts_ = com.google.protobuf.ByteString.EMPTY; } private byte memoizedIsInitialized = -1; public final boolean isInitialized() { @@ -276,20 +325,23 @@ public final boolean isInitialized() { public void writeTo(com.google.protobuf.CodedOutputStream output) throws java.io.IOException { getSerializedSize(); + for (int i = 0; i < guidePosts_.size(); i++) { + output.writeBytes(1, guidePosts_.get(i)); + } if (((bitField0_ & 0x00000001) == 0x00000001)) { - output.writeBytes(1, guidePosts_); + output.writeInt64(2, byteCount_); } if (((bitField0_ & 0x00000002) == 0x00000002)) { - output.writeInt64(2, byteCount_); + output.writeInt64(3, rowCount_); } if (((bitField0_ & 0x00000004) == 0x00000004)) { - output.writeInt64(3, rowCount_); + output.writeInt32(4, maxLength_); } if (((bitField0_ & 0x00000008) == 0x00000008)) { - output.writeInt32(4, maxLength_); + output.writeInt32(5, encodedGuidePostsCount_); } if (((bitField0_ & 0x00000010) == 0x00000010)) { - output.writeInt32(5, guidePostsCount_); + output.writeBytes(6, encodedGuidePosts_); } getUnknownFields().writeTo(output); } @@ -300,25 +352,34 @@ public int getSerializedSize() { if (size != -1) return size; size = 0; + { + int dataSize = 0; + for (int i = 0; i < guidePosts_.size(); i++) { + dataSize += com.google.protobuf.CodedOutputStream + .computeBytesSizeNoTag(guidePosts_.get(i)); + } + size += dataSize; + size += 1 * getGuidePostsList().size(); + } if (((bitField0_ & 0x00000001) == 0x00000001)) { size += com.google.protobuf.CodedOutputStream - .computeBytesSize(1, guidePosts_); + .computeInt64Size(2, byteCount_); } if (((bitField0_ & 0x00000002) == 0x00000002)) { size += com.google.protobuf.CodedOutputStream - .computeInt64Size(2, byteCount_); + .computeInt64Size(3, rowCount_); } if (((bitField0_ & 0x00000004) == 0x00000004)) { size += com.google.protobuf.CodedOutputStream - .computeInt64Size(3, rowCount_); + .computeInt32Size(4, maxLength_); } if (((bitField0_ & 0x00000008) == 0x00000008)) { size += com.google.protobuf.CodedOutputStream - .computeInt32Size(4, maxLength_); + .computeInt32Size(5, encodedGuidePostsCount_); } if (((bitField0_ & 0x00000010) == 0x00000010)) { size += com.google.protobuf.CodedOutputStream - .computeInt32Size(5, guidePostsCount_); + .computeBytesSize(6, encodedGuidePosts_); } size += getUnknownFields().getSerializedSize(); memoizedSerializedSize = size; @@ -343,11 +404,8 @@ public boolean equals(final java.lang.Object obj) { org.apache.phoenix.coprocessor.generated.PGuidePostsProtos.PGuidePosts other = (org.apache.phoenix.coprocessor.generated.PGuidePostsProtos.PGuidePosts) obj; boolean result = true; - result = result && (hasGuidePosts() == other.hasGuidePosts()); - if (hasGuidePosts()) { - result = result && getGuidePosts() - .equals(other.getGuidePosts()); - } + result = result && getGuidePostsList() + .equals(other.getGuidePostsList()); result = result && (hasByteCount() == other.hasByteCount()); if (hasByteCount()) { result = result && (getByteCount() @@ -363,10 +421,15 @@ public boolean equals(final java.lang.Object obj) { result = result && (getMaxLength() == other.getMaxLength()); } - result = result && (hasGuidePostsCount() == other.hasGuidePostsCount()); - if (hasGuidePostsCount()) { - result = result && (getGuidePostsCount() - == other.getGuidePostsCount()); + result = result && (hasEncodedGuidePostsCount() == other.hasEncodedGuidePostsCount()); + if (hasEncodedGuidePostsCount()) { + result = result && (getEncodedGuidePostsCount() + == other.getEncodedGuidePostsCount()); + } + result = result && (hasEncodedGuidePosts() == other.hasEncodedGuidePosts()); + if (hasEncodedGuidePosts()) { + result = result && getEncodedGuidePosts() + .equals(other.getEncodedGuidePosts()); } result = result && getUnknownFields().equals(other.getUnknownFields()); @@ -381,9 +444,9 @@ public int hashCode() { } int hash = 41; hash = (19 * hash) + getDescriptorForType().hashCode(); - if (hasGuidePosts()) { + if (getGuidePostsCount() > 0) { hash = (37 * hash) + GUIDEPOSTS_FIELD_NUMBER; - hash = (53 * hash) + getGuidePosts().hashCode(); + hash = (53 * hash) + getGuidePostsList().hashCode(); } if (hasByteCount()) { hash = (37 * hash) + BYTECOUNT_FIELD_NUMBER; @@ -397,9 +460,13 @@ public int hashCode() { hash = (37 * hash) + MAXLENGTH_FIELD_NUMBER; hash = (53 * hash) + getMaxLength(); } - if (hasGuidePostsCount()) { - hash = (37 * hash) + GUIDEPOSTSCOUNT_FIELD_NUMBER; - hash = (53 * hash) + getGuidePostsCount(); + if (hasEncodedGuidePostsCount()) { + hash = (37 * hash) + ENCODEDGUIDEPOSTSCOUNT_FIELD_NUMBER; + hash = (53 * hash) + getEncodedGuidePostsCount(); + } + if (hasEncodedGuidePosts()) { + hash = (37 * hash) + ENCODEDGUIDEPOSTS_FIELD_NUMBER; + hash = (53 * hash) + getEncodedGuidePosts().hashCode(); } hash = (29 * hash) + getUnknownFields().hashCode(); memoizedHashCode = hash; @@ -510,7 +577,7 @@ private static Builder create() { public Builder clear() { super.clear(); - guidePosts_ = com.google.protobuf.ByteString.EMPTY; + guidePosts_ = java.util.Collections.emptyList(); bitField0_ = (bitField0_ & ~0x00000001); byteCount_ = 0L; bitField0_ = (bitField0_ & ~0x00000002); @@ -518,8 +585,10 @@ public Builder clear() { bitField0_ = (bitField0_ & ~0x00000004); maxLength_ = 0; bitField0_ = (bitField0_ & ~0x00000008); - guidePostsCount_ = 0; + encodedGuidePostsCount_ = 0; bitField0_ = (bitField0_ & ~0x00000010); + encodedGuidePosts_ = com.google.protobuf.ByteString.EMPTY; + bitField0_ = (bitField0_ & ~0x00000020); return this; } @@ -548,26 +617,31 @@ public org.apache.phoenix.coprocessor.generated.PGuidePostsProtos.PGuidePosts bu org.apache.phoenix.coprocessor.generated.PGuidePostsProtos.PGuidePosts result = new org.apache.phoenix.coprocessor.generated.PGuidePostsProtos.PGuidePosts(this); int from_bitField0_ = bitField0_; int to_bitField0_ = 0; - if (((from_bitField0_ & 0x00000001) == 0x00000001)) { - to_bitField0_ |= 0x00000001; + if (((bitField0_ & 0x00000001) == 0x00000001)) { + guidePosts_ = java.util.Collections.unmodifiableList(guidePosts_); + bitField0_ = (bitField0_ & ~0x00000001); } result.guidePosts_ = guidePosts_; if (((from_bitField0_ & 0x00000002) == 0x00000002)) { - to_bitField0_ |= 0x00000002; + to_bitField0_ |= 0x00000001; } result.byteCount_ = byteCount_; if (((from_bitField0_ & 0x00000004) == 0x00000004)) { - to_bitField0_ |= 0x00000004; + to_bitField0_ |= 0x00000002; } result.rowCount_ = rowCount_; if (((from_bitField0_ & 0x00000008) == 0x00000008)) { - to_bitField0_ |= 0x00000008; + to_bitField0_ |= 0x00000004; } result.maxLength_ = maxLength_; if (((from_bitField0_ & 0x00000010) == 0x00000010)) { + to_bitField0_ |= 0x00000008; + } + result.encodedGuidePostsCount_ = encodedGuidePostsCount_; + if (((from_bitField0_ & 0x00000020) == 0x00000020)) { to_bitField0_ |= 0x00000010; } - result.guidePostsCount_ = guidePostsCount_; + result.encodedGuidePosts_ = encodedGuidePosts_; result.bitField0_ = to_bitField0_; onBuilt(); return result; @@ -584,8 +658,15 @@ public Builder mergeFrom(com.google.protobuf.Message other) { public Builder mergeFrom(org.apache.phoenix.coprocessor.generated.PGuidePostsProtos.PGuidePosts other) { if (other == org.apache.phoenix.coprocessor.generated.PGuidePostsProtos.PGuidePosts.getDefaultInstance()) return this; - if (other.hasGuidePosts()) { - setGuidePosts(other.getGuidePosts()); + if (!other.guidePosts_.isEmpty()) { + if (guidePosts_.isEmpty()) { + guidePosts_ = other.guidePosts_; + bitField0_ = (bitField0_ & ~0x00000001); + } else { + ensureGuidePostsIsMutable(); + guidePosts_.addAll(other.guidePosts_); + } + onChanged(); } if (other.hasByteCount()) { setByteCount(other.getByteCount()); @@ -596,8 +677,11 @@ public Builder mergeFrom(org.apache.phoenix.coprocessor.generated.PGuidePostsPro if (other.hasMaxLength()) { setMaxLength(other.getMaxLength()); } - if (other.hasGuidePostsCount()) { - setGuidePostsCount(other.getGuidePostsCount()); + if (other.hasEncodedGuidePostsCount()) { + setEncodedGuidePostsCount(other.getEncodedGuidePostsCount()); + } + if (other.hasEncodedGuidePosts()) { + setEncodedGuidePosts(other.getEncodedGuidePosts()); } this.mergeUnknownFields(other.getUnknownFields()); return this; @@ -626,38 +710,74 @@ public Builder mergeFrom( } private int bitField0_; - // optional bytes guidePosts = 1; - private com.google.protobuf.ByteString guidePosts_ = com.google.protobuf.ByteString.EMPTY; + // repeated bytes guidePosts = 1; + private java.util.List guidePosts_ = java.util.Collections.emptyList(); + private void ensureGuidePostsIsMutable() { + if (!((bitField0_ & 0x00000001) == 0x00000001)) { + guidePosts_ = new java.util.ArrayList(guidePosts_); + bitField0_ |= 0x00000001; + } + } /** - * optional bytes guidePosts = 1; + * repeated bytes guidePosts = 1; */ - public boolean hasGuidePosts() { - return ((bitField0_ & 0x00000001) == 0x00000001); + public java.util.List + getGuidePostsList() { + return java.util.Collections.unmodifiableList(guidePosts_); + } + /** + * repeated bytes guidePosts = 1; + */ + public int getGuidePostsCount() { + return guidePosts_.size(); } /** - * optional bytes guidePosts = 1; + * repeated bytes guidePosts = 1; */ - public com.google.protobuf.ByteString getGuidePosts() { - return guidePosts_; + public com.google.protobuf.ByteString getGuidePosts(int index) { + return guidePosts_.get(index); } /** - * optional bytes guidePosts = 1; + * repeated bytes guidePosts = 1; */ - public Builder setGuidePosts(com.google.protobuf.ByteString value) { + public Builder setGuidePosts( + int index, com.google.protobuf.ByteString value) { if (value == null) { throw new NullPointerException(); } - bitField0_ |= 0x00000001; - guidePosts_ = value; + ensureGuidePostsIsMutable(); + guidePosts_.set(index, value); onChanged(); return this; } /** - * optional bytes guidePosts = 1; + * repeated bytes guidePosts = 1; + */ + public Builder addGuidePosts(com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + ensureGuidePostsIsMutable(); + guidePosts_.add(value); + onChanged(); + return this; + } + /** + * repeated bytes guidePosts = 1; + */ + public Builder addAllGuidePosts( + java.lang.Iterable values) { + ensureGuidePostsIsMutable(); + super.addAll(values, guidePosts_); + onChanged(); + return this; + } + /** + * repeated bytes guidePosts = 1; */ public Builder clearGuidePosts() { + guidePosts_ = java.util.Collections.emptyList(); bitField0_ = (bitField0_ & ~0x00000001); - guidePosts_ = getDefaultInstance().getGuidePosts(); onChanged(); return this; } @@ -761,35 +881,71 @@ public Builder clearMaxLength() { return this; } - // optional int32 guidePostsCount = 5; - private int guidePostsCount_ ; + // optional int32 encodedGuidePostsCount = 5; + private int encodedGuidePostsCount_ ; /** - * optional int32 guidePostsCount = 5; + * optional int32 encodedGuidePostsCount = 5; */ - public boolean hasGuidePostsCount() { + public boolean hasEncodedGuidePostsCount() { return ((bitField0_ & 0x00000010) == 0x00000010); } /** - * optional int32 guidePostsCount = 5; + * optional int32 encodedGuidePostsCount = 5; */ - public int getGuidePostsCount() { - return guidePostsCount_; + public int getEncodedGuidePostsCount() { + return encodedGuidePostsCount_; } /** - * optional int32 guidePostsCount = 5; + * optional int32 encodedGuidePostsCount = 5; */ - public Builder setGuidePostsCount(int value) { + public Builder setEncodedGuidePostsCount(int value) { bitField0_ |= 0x00000010; - guidePostsCount_ = value; + encodedGuidePostsCount_ = value; onChanged(); return this; } /** - * optional int32 guidePostsCount = 5; + * optional int32 encodedGuidePostsCount = 5; */ - public Builder clearGuidePostsCount() { + public Builder clearEncodedGuidePostsCount() { bitField0_ = (bitField0_ & ~0x00000010); - guidePostsCount_ = 0; + encodedGuidePostsCount_ = 0; + onChanged(); + return this; + } + + // optional bytes encodedGuidePosts = 6; + private com.google.protobuf.ByteString encodedGuidePosts_ = com.google.protobuf.ByteString.EMPTY; + /** + * optional bytes encodedGuidePosts = 6; + */ + public boolean hasEncodedGuidePosts() { + return ((bitField0_ & 0x00000020) == 0x00000020); + } + /** + * optional bytes encodedGuidePosts = 6; + */ + public com.google.protobuf.ByteString getEncodedGuidePosts() { + return encodedGuidePosts_; + } + /** + * optional bytes encodedGuidePosts = 6; + */ + public Builder setEncodedGuidePosts(com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000020; + encodedGuidePosts_ = value; + onChanged(); + return this; + } + /** + * optional bytes encodedGuidePosts = 6; + */ + public Builder clearEncodedGuidePosts() { + bitField0_ = (bitField0_ & ~0x00000020); + encodedGuidePosts_ = getDefaultInstance().getEncodedGuidePosts(); onChanged(); return this; } @@ -819,12 +975,13 @@ public Builder clearGuidePostsCount() { descriptor; static { java.lang.String[] descriptorData = { - "\n\021PGuidePosts.proto\"r\n\013PGuidePosts\022\022\n\ngu" + - "idePosts\030\001 \001(\014\022\021\n\tbyteCount\030\002 \001(\003\022\020\n\010row" + - "Count\030\003 \001(\003\022\021\n\tmaxLength\030\004 \001(\005\022\027\n\017guideP" + - "ostsCount\030\005 \001(\005BE\n(org.apache.phoenix.co" + - "processor.generatedB\021PGuidePostsProtosH\001" + - "\210\001\001\240\001\001" + "\n\021PGuidePosts.proto\"\224\001\n\013PGuidePosts\022\022\n\ng" + + "uidePosts\030\001 \003(\014\022\021\n\tbyteCount\030\002 \001(\003\022\020\n\010ro" + + "wCount\030\003 \001(\003\022\021\n\tmaxLength\030\004 \001(\005\022\036\n\026encod" + + "edGuidePostsCount\030\005 \001(\005\022\031\n\021encodedGuideP" + + "osts\030\006 \001(\014BE\n(org.apache.phoenix.coproce" + + "ssor.generatedB\021PGuidePostsProtosH\001\210\001\001\240\001" + + "\001" }; com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner = new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() { @@ -836,7 +993,7 @@ public com.google.protobuf.ExtensionRegistry assignDescriptors( internal_static_PGuidePosts_fieldAccessorTable = new com.google.protobuf.GeneratedMessage.FieldAccessorTable( internal_static_PGuidePosts_descriptor, - new java.lang.String[] { "GuidePosts", "ByteCount", "RowCount", "MaxLength", "GuidePostsCount", }); + new java.lang.String[] { "GuidePosts", "ByteCount", "RowCount", "MaxLength", "EncodedGuidePostsCount", "EncodedGuidePosts", }); return null; } }; diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/PTableProtos.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/PTableProtos.java index 28f5d24829a..1621f36ce5e 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/PTableProtos.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/PTableProtos.java @@ -1901,15 +1901,15 @@ public interface PTableStatsOrBuilder */ org.apache.phoenix.coprocessor.generated.PGuidePostsProtos.PGuidePostsOrBuilder getPGuidePostsOrBuilder(); - // optional bytes guidePosts = 7; + // optional bytes encodedGuidePosts = 7; /** - * optional bytes guidePosts = 7; + * optional bytes encodedGuidePosts = 7; */ - boolean hasGuidePosts(); + boolean hasEncodedGuidePosts(); /** - * optional bytes guidePosts = 7; + * optional bytes encodedGuidePosts = 7; */ - com.google.protobuf.ByteString getGuidePosts(); + com.google.protobuf.ByteString getEncodedGuidePosts(); // optional int32 maxLength = 8; /** @@ -2015,7 +2015,7 @@ private PTableStats( } case 58: { bitField0_ |= 0x00000020; - guidePosts_ = input.readBytes(); + encodedGuidePosts_ = input.readBytes(); break; } case 64: { @@ -2175,20 +2175,20 @@ public org.apache.phoenix.coprocessor.generated.PGuidePostsProtos.PGuidePostsOrB return pGuidePosts_; } - // optional bytes guidePosts = 7; - public static final int GUIDEPOSTS_FIELD_NUMBER = 7; - private com.google.protobuf.ByteString guidePosts_; + // optional bytes encodedGuidePosts = 7; + public static final int ENCODEDGUIDEPOSTS_FIELD_NUMBER = 7; + private com.google.protobuf.ByteString encodedGuidePosts_; /** - * optional bytes guidePosts = 7; + * optional bytes encodedGuidePosts = 7; */ - public boolean hasGuidePosts() { + public boolean hasEncodedGuidePosts() { return ((bitField0_ & 0x00000020) == 0x00000020); } /** - * optional bytes guidePosts = 7; + * optional bytes encodedGuidePosts = 7; */ - public com.google.protobuf.ByteString getGuidePosts() { - return guidePosts_; + public com.google.protobuf.ByteString getEncodedGuidePosts() { + return encodedGuidePosts_; } // optional int32 maxLength = 8; @@ -2214,7 +2214,7 @@ private void initFields() { keyBytesCount_ = 0L; guidePostsCount_ = 0; pGuidePosts_ = org.apache.phoenix.coprocessor.generated.PGuidePostsProtos.PGuidePosts.getDefaultInstance(); - guidePosts_ = com.google.protobuf.ByteString.EMPTY; + encodedGuidePosts_ = com.google.protobuf.ByteString.EMPTY; maxLength_ = 0; } private byte memoizedIsInitialized = -1; @@ -2252,7 +2252,7 @@ public void writeTo(com.google.protobuf.CodedOutputStream output) output.writeMessage(6, pGuidePosts_); } if (((bitField0_ & 0x00000020) == 0x00000020)) { - output.writeBytes(7, guidePosts_); + output.writeBytes(7, encodedGuidePosts_); } if (((bitField0_ & 0x00000040) == 0x00000040)) { output.writeInt32(8, maxLength_); @@ -2297,7 +2297,7 @@ public int getSerializedSize() { } if (((bitField0_ & 0x00000020) == 0x00000020)) { size += com.google.protobuf.CodedOutputStream - .computeBytesSize(7, guidePosts_); + .computeBytesSize(7, encodedGuidePosts_); } if (((bitField0_ & 0x00000040) == 0x00000040)) { size += com.google.protobuf.CodedOutputStream @@ -2353,10 +2353,10 @@ public boolean equals(final java.lang.Object obj) { result = result && getPGuidePosts() .equals(other.getPGuidePosts()); } - result = result && (hasGuidePosts() == other.hasGuidePosts()); - if (hasGuidePosts()) { - result = result && getGuidePosts() - .equals(other.getGuidePosts()); + result = result && (hasEncodedGuidePosts() == other.hasEncodedGuidePosts()); + if (hasEncodedGuidePosts()) { + result = result && getEncodedGuidePosts() + .equals(other.getEncodedGuidePosts()); } result = result && (hasMaxLength() == other.hasMaxLength()); if (hasMaxLength()) { @@ -2400,9 +2400,9 @@ public int hashCode() { hash = (37 * hash) + PGUIDEPOSTS_FIELD_NUMBER; hash = (53 * hash) + getPGuidePosts().hashCode(); } - if (hasGuidePosts()) { - hash = (37 * hash) + GUIDEPOSTS_FIELD_NUMBER; - hash = (53 * hash) + getGuidePosts().hashCode(); + if (hasEncodedGuidePosts()) { + hash = (37 * hash) + ENCODEDGUIDEPOSTS_FIELD_NUMBER; + hash = (53 * hash) + getEncodedGuidePosts().hashCode(); } if (hasMaxLength()) { hash = (37 * hash) + MAXLENGTH_FIELD_NUMBER; @@ -2534,7 +2534,7 @@ public Builder clear() { pGuidePostsBuilder_.clear(); } bitField0_ = (bitField0_ & ~0x00000020); - guidePosts_ = com.google.protobuf.ByteString.EMPTY; + encodedGuidePosts_ = com.google.protobuf.ByteString.EMPTY; bitField0_ = (bitField0_ & ~0x00000040); maxLength_ = 0; bitField0_ = (bitField0_ & ~0x00000080); @@ -2598,7 +2598,7 @@ public org.apache.phoenix.coprocessor.generated.PTableProtos.PTableStats buildPa if (((from_bitField0_ & 0x00000040) == 0x00000040)) { to_bitField0_ |= 0x00000020; } - result.guidePosts_ = guidePosts_; + result.encodedGuidePosts_ = encodedGuidePosts_; if (((from_bitField0_ & 0x00000080) == 0x00000080)) { to_bitField0_ |= 0x00000040; } @@ -2644,8 +2644,8 @@ public Builder mergeFrom(org.apache.phoenix.coprocessor.generated.PTableProtos.P if (other.hasPGuidePosts()) { mergePGuidePosts(other.getPGuidePosts()); } - if (other.hasGuidePosts()) { - setGuidePosts(other.getGuidePosts()); + if (other.hasEncodedGuidePosts()) { + setEncodedGuidePosts(other.getEncodedGuidePosts()); } if (other.hasMaxLength()) { setMaxLength(other.getMaxLength()); @@ -3005,38 +3005,38 @@ public org.apache.phoenix.coprocessor.generated.PGuidePostsProtos.PGuidePostsOrB return pGuidePostsBuilder_; } - // optional bytes guidePosts = 7; - private com.google.protobuf.ByteString guidePosts_ = com.google.protobuf.ByteString.EMPTY; + // optional bytes encodedGuidePosts = 7; + private com.google.protobuf.ByteString encodedGuidePosts_ = com.google.protobuf.ByteString.EMPTY; /** - * optional bytes guidePosts = 7; + * optional bytes encodedGuidePosts = 7; */ - public boolean hasGuidePosts() { + public boolean hasEncodedGuidePosts() { return ((bitField0_ & 0x00000040) == 0x00000040); } /** - * optional bytes guidePosts = 7; + * optional bytes encodedGuidePosts = 7; */ - public com.google.protobuf.ByteString getGuidePosts() { - return guidePosts_; + public com.google.protobuf.ByteString getEncodedGuidePosts() { + return encodedGuidePosts_; } /** - * optional bytes guidePosts = 7; + * optional bytes encodedGuidePosts = 7; */ - public Builder setGuidePosts(com.google.protobuf.ByteString value) { + public Builder setEncodedGuidePosts(com.google.protobuf.ByteString value) { if (value == null) { throw new NullPointerException(); } bitField0_ |= 0x00000040; - guidePosts_ = value; + encodedGuidePosts_ = value; onChanged(); return this; } /** - * optional bytes guidePosts = 7; + * optional bytes encodedGuidePosts = 7; */ - public Builder clearGuidePosts() { + public Builder clearEncodedGuidePosts() { bitField0_ = (bitField0_ & ~0x00000040); - guidePosts_ = getDefaultInstance().getGuidePosts(); + encodedGuidePosts_ = getDefaultInstance().getEncodedGuidePosts(); onChanged(); return this; } @@ -7099,6 +7099,7 @@ public Builder clearUpdateCacheFrequency() { "\006 \002(\010\022\020\n\010position\030\007 \002(\005\022\021\n\tsortOrder\030\010 \002" + "(\005\022\021\n\tarraySize\030\t \001(\005\022\024\n\014viewConstant\030\n " + "\001(\014\022\026\n\016viewReferenced\030\013 \001(\010\022\022\n\nexpressio" + +<<<<<<< 5591b5a7dbbe8f086523c8d1b53b4a913a516896 <<<<<<< 056ddbed5ca75018213b02d599c7c7a93e8a6692 <<<<<<< 3520e12858223dfcad343e0a3a29c71fe4d074bc "n\030\014 \001(\t\022\026\n\016isRowTimestamp\030\r \001(\010\"\232\001\n\013PTab" + @@ -7181,6 +7182,34 @@ public Builder clearUpdateCacheFrequency() { "oenix.coprocessor.generatedB\014PTableProto" + "sH\001\210\001\001\240\001\001" >>>>>>> review comments second round +======= + "n\030\014 \001(\t\022\026\n\016isRowTimestamp\030\r \001(\010\"\310\001\n\013PTab" + + "leStats\022\013\n\003key\030\001 \002(\014\022\016\n\006values\030\002 \003(\014\022\033\n\023" + + "guidePostsByteCount\030\003 \001(\003\022\025\n\rkeyBytesCou", + "nt\030\004 \001(\003\022\027\n\017guidePostsCount\030\005 \001(\005\022!\n\013pGu" + + "idePosts\030\006 \001(\0132\014.PGuidePosts\022\031\n\021encodedG" + + "uidePosts\030\007 \001(\014\022\021\n\tmaxLength\030\010 \001(\005\"\206\005\n\006P" + + "Table\022\027\n\017schemaNameBytes\030\001 \002(\014\022\026\n\016tableN" + + "ameBytes\030\002 \002(\014\022\036\n\ttableType\030\003 \002(\0162\013.PTab" + + "leType\022\022\n\nindexState\030\004 \001(\t\022\026\n\016sequenceNu" + + "mber\030\005 \002(\003\022\021\n\ttimeStamp\030\006 \002(\003\022\023\n\013pkNameB" + + "ytes\030\007 \001(\014\022\021\n\tbucketNum\030\010 \002(\005\022\031\n\007columns" + + "\030\t \003(\0132\010.PColumn\022\030\n\007indexes\030\n \003(\0132\007.PTab" + + "le\022\027\n\017isImmutableRows\030\013 \002(\010\022 \n\nguidePost", + "s\030\014 \003(\0132\014.PTableStats\022\032\n\022dataTableNameBy" + + "tes\030\r \001(\014\022\031\n\021defaultFamilyName\030\016 \001(\014\022\022\n\n" + + "disableWAL\030\017 \002(\010\022\023\n\013multiTenant\030\020 \002(\010\022\020\n" + + "\010viewType\030\021 \001(\014\022\025\n\rviewStatement\030\022 \001(\014\022\025" + + "\n\rphysicalNames\030\023 \003(\014\022\020\n\010tenantId\030\024 \001(\014\022" + + "\023\n\013viewIndexId\030\025 \001(\005\022\021\n\tindexType\030\026 \001(\014\022" + + "\026\n\016statsTimeStamp\030\027 \001(\003\022\022\n\nstoreNulls\030\030 " + + "\001(\010\022\027\n\017baseColumnCount\030\031 \001(\005\022\036\n\026rowKeyOr" + + "derOptimizable\030\032 \001(\010\022\025\n\rtransactional\030\033 " + + "\001(\010*A\n\nPTableType\022\n\n\006SYSTEM\020\000\022\010\n\004USER\020\001\022", + "\010\n\004VIEW\020\002\022\t\n\005INDEX\020\003\022\010\n\004JOIN\020\004B@\n(org.ap" + + "ache.phoenix.coprocessor.generatedB\014PTab" + + "leProtosH\001\210\001\001\240\001\001" +>>>>>>> updatd review comments for protobuf backward compatibility and moving code to upgrade Util }; com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner = new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() { @@ -7198,7 +7227,7 @@ public com.google.protobuf.ExtensionRegistry assignDescriptors( internal_static_PTableStats_fieldAccessorTable = new com.google.protobuf.GeneratedMessage.FieldAccessorTable( internal_static_PTableStats_descriptor, - new java.lang.String[] { "Key", "Values", "GuidePostsByteCount", "KeyBytesCount", "GuidePostsCount", "PGuidePosts", "GuidePosts", "MaxLength", }); + new java.lang.String[] { "Key", "Values", "GuidePostsByteCount", "KeyBytesCount", "GuidePostsCount", "PGuidePosts", "EncodedGuidePosts", "MaxLength", }); internal_static_PTable_descriptor = getDescriptor().getMessageTypes().get(2); internal_static_PTable_fieldAccessorTable = new diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java index 7df5b1c5e9f..a7260fb79f9 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java @@ -1022,9 +1022,9 @@ public static PTable createFromProto(PTableProtos.PTable table) { long guidePostsByteCount = pGuidePosts.getByteCount(); long rowCount = pGuidePosts.getRowCount(); int maxLength = pGuidePosts.getMaxLength(); - int guidePostsCount = pGuidePosts.getGuidePostsCount(); + int guidePostsCount = pGuidePosts.getEncodedGuidePostsCount(); GuidePostsInfo info = new GuidePostsInfo(guidePostsByteCount, - new ImmutableBytesWritable(HBaseZeroCopyByteString.zeroCopyGetBytes(pGuidePosts.getGuidePosts())), rowCount, maxLength, guidePostsCount); + new ImmutableBytesWritable(HBaseZeroCopyByteString.zeroCopyGetBytes(pGuidePosts.getEncodedGuidePosts())), rowCount, maxLength, guidePostsCount); tableGuidePosts.put(pTableStatsProto.getKey().toByteArray(), info); } PTableStats stats = new PTableStatsImpl(tableGuidePosts, table.getStatsTimeStamp()); @@ -1132,16 +1132,16 @@ public static PTableProtos.PTable toProto(PTable table) { for (Map.Entry entry : table.getTableStats().getGuidePosts().entrySet()) { PTableProtos.PTableStats.Builder statsBuilder = PTableProtos.PTableStats.newBuilder(); statsBuilder.setKey(ByteStringer.wrap(entry.getKey())); - statsBuilder.setGuidePosts(ByteStringer.wrap(entry.getValue().getGuidePosts().get())); + statsBuilder.setEncodedGuidePosts(ByteStringer.wrap(entry.getValue().getGuidePosts().get())); statsBuilder.setGuidePostsByteCount(entry.getValue().getByteCount()); statsBuilder.setMaxLength(entry.getValue().getMaxLength()); statsBuilder.setGuidePostsCount(entry.getValue().getGuidePostsCount()); PGuidePostsProtos.PGuidePosts.Builder guidePstsBuilder = PGuidePostsProtos.PGuidePosts.newBuilder(); - guidePstsBuilder.setGuidePosts(ByteStringer.wrap(entry.getValue().getGuidePosts().get())); + guidePstsBuilder.setEncodedGuidePosts(ByteStringer.wrap(entry.getValue().getGuidePosts().get())); guidePstsBuilder.setByteCount(entry.getValue().getByteCount()); guidePstsBuilder.setRowCount(entry.getValue().getRowCount()); guidePstsBuilder.setMaxLength(entry.getValue().getMaxLength()); - guidePstsBuilder.setGuidePostsCount(entry.getValue().getGuidePostsCount()); + guidePstsBuilder.setEncodedGuidePostsCount(entry.getValue().getGuidePostsCount()); statsBuilder.setPGuidePosts(guidePstsBuilder); builder.addGuidePosts(statsBuilder.build()); } diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/UpgradeUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/UpgradeUtil.java index c9318512975..4d6afcea50d 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/util/UpgradeUtil.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/util/UpgradeUtil.java @@ -52,12 +52,14 @@ import javax.annotation.Nonnull; import javax.annotation.Nullable; +import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.Delete; +import org.apache.hadoop.hbase.client.Get; import org.apache.hadoop.hbase.client.HBaseAdmin; import org.apache.hadoop.hbase.client.HTableInterface; import org.apache.hadoop.hbase.client.Mutation; @@ -100,6 +102,7 @@ public class UpgradeUtil { private static final Logger logger = LoggerFactory.getLogger(UpgradeUtil.class); private static final byte[] SEQ_PREFIX_BYTES = ByteUtil.concat(QueryConstants.SEPARATOR_BYTE_ARRAY, Bytes.toBytes("_SEQ_")); + public static final byte[] UPGRADE_TO_4_7_COLUMN_NAME = Bytes.toBytes("UPGRADE_TO_4_7"); public static String UPSERT_BASE_COLUMN_COUNT_IN_HEADER_ROW = "UPSERT " + "INTO SYSTEM.CATALOG " @@ -1215,4 +1218,67 @@ public static void addRowKeyOrderOptimizableCell(List tableMetadata, b MetaDataEndpointImpl.ROW_KEY_ORDER_OPTIMIZABLE_BYTES, PBoolean.INSTANCE.toBytes(true)); tableMetadata.add(put); } + + public static boolean truncateStats(HTableInterface metaTable, HTableInterface statsTable) + throws IOException, InterruptedException { + List columnCells = metaTable + .get(new Get(SchemaUtil.getTableKey(null, PhoenixDatabaseMetaData.SYSTEM_SCHEMA_NAME, + PhoenixDatabaseMetaData.SYSTEM_CATALOG_TABLE))) + .getColumnCells(PhoenixDatabaseMetaData.TABLE_FAMILY_BYTES, QueryConstants.EMPTY_COLUMN_BYTES); + if (!columnCells.isEmpty() + && columnCells.get(0).getTimestamp() < MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_7_0) { + + byte[] statsTableKey = SchemaUtil.getTableKey(null, PhoenixDatabaseMetaData.SYSTEM_SCHEMA_NAME, + PhoenixDatabaseMetaData.SYSTEM_STATS_TABLE); + KeyValue upgradeKV = KeyValueUtil.newKeyValue(statsTableKey, PhoenixDatabaseMetaData.TABLE_FAMILY_BYTES, + UPGRADE_TO_4_7_COLUMN_NAME, MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_7_0 - 1, + ByteUtil.EMPTY_BYTE_ARRAY); + Put upgradePut = new Put(statsTableKey); + upgradePut.add(upgradeKV); + + // check for null in UPGRADE_TO_4_7_COLUMN_NAME in checkAndPut so that only single client + // drop the rows of SYSTEM.STATS + if (metaTable.checkAndPut(statsTableKey, PhoenixDatabaseMetaData.TABLE_FAMILY_BYTES, + UPGRADE_TO_4_7_COLUMN_NAME, null, upgradePut)) { + List mutations = Lists.newArrayListWithExpectedSize(1000); + Scan scan = new Scan(); + scan.setRaw(true); + scan.setMaxVersions(); + ResultScanner statsScanner = statsTable.getScanner(scan); + Result r; + mutations.clear(); + int count = 0; + while ((r = statsScanner.next()) != null) { + Delete delete = null; + for (KeyValue keyValue : r.raw()) { + if (KeyValue.Type.codeToType(keyValue.getType()) == KeyValue.Type.Put) { + if (delete == null) { + delete = new Delete(keyValue.getRow()); + } + KeyValue deleteKeyValue = new KeyValue(keyValue.getRowArray(), keyValue.getRowOffset(), + keyValue.getRowLength(), keyValue.getFamilyArray(), keyValue.getFamilyOffset(), + keyValue.getFamilyLength(), keyValue.getQualifierArray(), + keyValue.getQualifierOffset(), keyValue.getQualifierLength(), + keyValue.getTimestamp(), KeyValue.Type.Delete, ByteUtil.EMPTY_BYTE_ARRAY, 0, 0); + delete.addDeleteMarker(deleteKeyValue); + } + } + if (delete != null) { + mutations.add(delete); + if (count > 10) { + statsTable.batch(mutations); + mutations.clear(); + count = 0; + } + count++; + } + } + if (!mutations.isEmpty()) { + statsTable.batch(mutations); + } + return true; + } + } + return false; + } } diff --git a/phoenix-protocol/src/main/PGuidePosts.proto b/phoenix-protocol/src/main/PGuidePosts.proto index d0c7c9bdfd7..14de2ebc9e1 100644 --- a/phoenix-protocol/src/main/PGuidePosts.proto +++ b/phoenix-protocol/src/main/PGuidePosts.proto @@ -24,9 +24,10 @@ option java_generic_services = true; option java_generate_equals_and_hash = true; option optimize_for = SPEED; message PGuidePosts { - optional bytes guidePosts = 1; + repeated bytes guidePosts = 1; optional int64 byteCount = 2; optional int64 rowCount = 3; optional int32 maxLength = 4; - optional int32 guidePostsCount = 5; + optional int32 encodedGuidePostsCount = 5; + optional bytes encodedGuidePosts = 6; } \ No newline at end of file diff --git a/phoenix-protocol/src/main/PTable.proto b/phoenix-protocol/src/main/PTable.proto index cc81d627d73..0b6b763b2bc 100644 --- a/phoenix-protocol/src/main/PTable.proto +++ b/phoenix-protocol/src/main/PTable.proto @@ -57,7 +57,7 @@ message PTableStats { optional int64 keyBytesCount = 4; optional int32 guidePostsCount = 5; optional PGuidePosts pGuidePosts = 6; - optional bytes guidePosts = 7; + optional bytes encodedGuidePosts = 7; optional int32 maxLength = 8; } From cdbf95e734c1eeed03d77d52162cf05695d6d844 Mon Sep 17 00:00:00 2001 From: Ankit Singhal Date: Wed, 20 Jan 2016 16:22:23 +0530 Subject: [PATCH 47/47] rebase changes --- .../coprocessor/generated/PTableProtos.java | 96 ++----------------- 1 file changed, 6 insertions(+), 90 deletions(-) diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/PTableProtos.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/PTableProtos.java index 1621f36ce5e..11a9049ae44 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/PTableProtos.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/PTableProtos.java @@ -7099,96 +7099,12 @@ public Builder clearUpdateCacheFrequency() { "\006 \002(\010\022\020\n\010position\030\007 \002(\005\022\021\n\tsortOrder\030\010 \002" + "(\005\022\021\n\tarraySize\030\t \001(\005\022\024\n\014viewConstant\030\n " + "\001(\014\022\026\n\016viewReferenced\030\013 \001(\010\022\022\n\nexpressio" + -<<<<<<< 5591b5a7dbbe8f086523c8d1b53b4a913a516896 -<<<<<<< 056ddbed5ca75018213b02d599c7c7a93e8a6692 -<<<<<<< 3520e12858223dfcad343e0a3a29c71fe4d074bc - "n\030\014 \001(\t\022\026\n\016isRowTimestamp\030\r \001(\010\"\232\001\n\013PTab" + - "leStats\022\013\n\003key\030\001 \002(\014\022\016\n\006values\030\002 \003(\014\022\033\n\023" + - "guidePostsByteCount\030\003 \001(\003\022\025\n\rkeyBytesCou", - "nt\030\004 \001(\003\022\027\n\017guidePostsCount\030\005 \001(\005\022!\n\013pGu" + - "idePosts\030\006 \001(\0132\014.PGuidePosts\"\244\005\n\006PTable\022" + - "\027\n\017schemaNameBytes\030\001 \002(\014\022\026\n\016tableNameByt" + - "es\030\002 \002(\014\022\036\n\ttableType\030\003 \002(\0162\013.PTableType" + - "\022\022\n\nindexState\030\004 \001(\t\022\026\n\016sequenceNumber\030\005" + - " \002(\003\022\021\n\ttimeStamp\030\006 \002(\003\022\023\n\013pkNameBytes\030\007" + - " \001(\014\022\021\n\tbucketNum\030\010 \002(\005\022\031\n\007columns\030\t \003(\013" + - "2\010.PColumn\022\030\n\007indexes\030\n \003(\0132\007.PTable\022\027\n\017" + - "isImmutableRows\030\013 \002(\010\022 \n\nguidePosts\030\014 \003(" + - "\0132\014.PTableStats\022\032\n\022dataTableNameBytes\030\r ", - "\001(\014\022\031\n\021defaultFamilyName\030\016 \001(\014\022\022\n\ndisabl" + - "eWAL\030\017 \002(\010\022\023\n\013multiTenant\030\020 \002(\010\022\020\n\010viewT" + - "ype\030\021 \001(\014\022\025\n\rviewStatement\030\022 \001(\014\022\025\n\rphys" + - "icalNames\030\023 \003(\014\022\020\n\010tenantId\030\024 \001(\014\022\023\n\013vie" + - "wIndexId\030\025 \001(\005\022\021\n\tindexType\030\026 \001(\014\022\026\n\016sta" + - "tsTimeStamp\030\027 \001(\003\022\022\n\nstoreNulls\030\030 \001(\010\022\027\n" + - "\017baseColumnCount\030\031 \001(\005\022\036\n\026rowKeyOrderOpt" + - "imizable\030\032 \001(\010\022\025\n\rtransactional\030\033 \001(\010\022\034\n" + - "\024updateCacheFrequency\030\034 \001(\003*A\n\nPTableTyp" + - "e\022\n\n\006SYSTEM\020\000\022\010\n\004USER\020\001\022\010\n\004VIEW\020\002\022\t\n\005IND", - "EX\020\003\022\010\n\004JOIN\020\004B@\n(org.apache.phoenix.cop" + - "rocessor.generatedB\014PTableProtosH\001\210\001\001\240\001\001" -======= - "n\030\014 \001(\t\022\026\n\016isRowTimestamp\030\r \001(\010\"\261\001\n\013PTab" + - "leStats\022\013\n\003key\030\001 \002(\014\022\022\n\nguidePosts\030\002 \001(\014" + - "\022\033\n\023guidePostsByteCount\030\003 \001(\003\022\025\n\rkeyByte", - "sCount\030\004 \001(\003\022\027\n\017guidePostsCount\030\005 \001(\005\022!\n" + - "\013pGuidePosts\030\006 \001(\0132\014.PGuidePosts\022\021\n\tmaxL" + - "ength\030\007 \001(\005\"\206\005\n\006PTable\022\027\n\017schemaNameByte" + - "s\030\001 \002(\014\022\026\n\016tableNameBytes\030\002 \002(\014\022\036\n\ttable" + - "Type\030\003 \002(\0162\013.PTableType\022\022\n\nindexState\030\004 " + - "\001(\t\022\026\n\016sequenceNumber\030\005 \002(\003\022\021\n\ttimeStamp" + - "\030\006 \002(\003\022\023\n\013pkNameBytes\030\007 \001(\014\022\021\n\tbucketNum" + - "\030\010 \002(\005\022\031\n\007columns\030\t \003(\0132\010.PColumn\022\030\n\007ind" + - "exes\030\n \003(\0132\007.PTable\022\027\n\017isImmutableRows\030\013" + - " \002(\010\022 \n\nguidePosts\030\014 \003(\0132\014.PTableStats\022\032", - "\n\022dataTableNameBytes\030\r \001(\014\022\031\n\021defaultFam" + - "ilyName\030\016 \001(\014\022\022\n\ndisableWAL\030\017 \002(\010\022\023\n\013mul" + - "tiTenant\030\020 \002(\010\022\020\n\010viewType\030\021 \001(\014\022\025\n\rview" + - "Statement\030\022 \001(\014\022\025\n\rphysicalNames\030\023 \003(\014\022\020" + - "\n\010tenantId\030\024 \001(\014\022\023\n\013viewIndexId\030\025 \001(\005\022\021\n" + - "\tindexType\030\026 \001(\014\022\026\n\016statsTimeStamp\030\027 \001(\003" + - "\022\022\n\nstoreNulls\030\030 \001(\010\022\027\n\017baseColumnCount\030" + - "\031 \001(\005\022\036\n\026rowKeyOrderOptimizable\030\032 \001(\010\022\025\n" + - "\rtransactional\030\033 \001(\010*A\n\nPTableType\022\n\n\006SY" + - "STEM\020\000\022\010\n\004USER\020\001\022\010\n\004VIEW\020\002\022\t\n\005INDEX\020\003\022\010\n", - "\004JOIN\020\004B@\n(org.apache.phoenix.coprocesso" + - "r.generatedB\014PTableProtosH\001\210\001\001\240\001\001" ->>>>>>> PHOENIX-2417 Compress memory used by row key byte[] of guideposts -======= - "n\030\014 \001(\t\022\026\n\016isRowTimestamp\030\r \001(\010\"\301\001\n\013PTab" + - "leStats\022\013\n\003key\030\001 \002(\014\022\016\n\006values\030\002 \003(\014\022\033\n\023" + - "guidePostsByteCount\030\003 \001(\003\022\025\n\rkeyBytesCou", - "nt\030\004 \001(\003\022\027\n\017guidePostsCount\030\005 \001(\005\022!\n\013pGu" + - "idePosts\030\006 \001(\0132\014.PGuidePosts\022\022\n\nguidePos" + - "ts\030\007 \001(\014\022\021\n\tmaxLength\030\010 \001(\005\"\206\005\n\006PTable\022\027" + - "\n\017schemaNameBytes\030\001 \002(\014\022\026\n\016tableNameByte" + - "s\030\002 \002(\014\022\036\n\ttableType\030\003 \002(\0162\013.PTableType\022" + - "\022\n\nindexState\030\004 \001(\t\022\026\n\016sequenceNumber\030\005 " + - "\002(\003\022\021\n\ttimeStamp\030\006 \002(\003\022\023\n\013pkNameBytes\030\007 " + - "\001(\014\022\021\n\tbucketNum\030\010 \002(\005\022\031\n\007columns\030\t \003(\0132" + - "\010.PColumn\022\030\n\007indexes\030\n \003(\0132\007.PTable\022\027\n\017i" + - "sImmutableRows\030\013 \002(\010\022 \n\nguidePosts\030\014 \003(\013", - "2\014.PTableStats\022\032\n\022dataTableNameBytes\030\r \001" + - "(\014\022\031\n\021defaultFamilyName\030\016 \001(\014\022\022\n\ndisable" + - "WAL\030\017 \002(\010\022\023\n\013multiTenant\030\020 \002(\010\022\020\n\010viewTy" + - "pe\030\021 \001(\014\022\025\n\rviewStatement\030\022 \001(\014\022\025\n\rphysi" + - "calNames\030\023 \003(\014\022\020\n\010tenantId\030\024 \001(\014\022\023\n\013view" + - "IndexId\030\025 \001(\005\022\021\n\tindexType\030\026 \001(\014\022\026\n\016stat" + - "sTimeStamp\030\027 \001(\003\022\022\n\nstoreNulls\030\030 \001(\010\022\027\n\017" + - "baseColumnCount\030\031 \001(\005\022\036\n\026rowKeyOrderOpti" + - "mizable\030\032 \001(\010\022\025\n\rtransactional\030\033 \001(\010*A\n\n" + - "PTableType\022\n\n\006SYSTEM\020\000\022\010\n\004USER\020\001\022\010\n\004VIEW", - "\020\002\022\t\n\005INDEX\020\003\022\010\n\004JOIN\020\004B@\n(org.apache.ph" + - "oenix.coprocessor.generatedB\014PTableProto" + - "sH\001\210\001\001\240\001\001" ->>>>>>> review comments second round -======= "n\030\014 \001(\t\022\026\n\016isRowTimestamp\030\r \001(\010\"\310\001\n\013PTab" + "leStats\022\013\n\003key\030\001 \002(\014\022\016\n\006values\030\002 \003(\014\022\033\n\023" + "guidePostsByteCount\030\003 \001(\003\022\025\n\rkeyBytesCou", "nt\030\004 \001(\003\022\027\n\017guidePostsCount\030\005 \001(\005\022!\n\013pGu" + "idePosts\030\006 \001(\0132\014.PGuidePosts\022\031\n\021encodedG" + - "uidePosts\030\007 \001(\014\022\021\n\tmaxLength\030\010 \001(\005\"\206\005\n\006P" + + "uidePosts\030\007 \001(\014\022\021\n\tmaxLength\030\010 \001(\005\"\244\005\n\006P" + "Table\022\027\n\017schemaNameBytes\030\001 \002(\014\022\026\n\016tableN" + "ameBytes\030\002 \002(\014\022\036\n\ttableType\030\003 \002(\0162\013.PTab" + "leType\022\022\n\nindexState\030\004 \001(\t\022\026\n\016sequenceNu" + @@ -7205,11 +7121,11 @@ public Builder clearUpdateCacheFrequency() { "\026\n\016statsTimeStamp\030\027 \001(\003\022\022\n\nstoreNulls\030\030 " + "\001(\010\022\027\n\017baseColumnCount\030\031 \001(\005\022\036\n\026rowKeyOr" + "derOptimizable\030\032 \001(\010\022\025\n\rtransactional\030\033 " + - "\001(\010*A\n\nPTableType\022\n\n\006SYSTEM\020\000\022\010\n\004USER\020\001\022", - "\010\n\004VIEW\020\002\022\t\n\005INDEX\020\003\022\010\n\004JOIN\020\004B@\n(org.ap" + - "ache.phoenix.coprocessor.generatedB\014PTab" + - "leProtosH\001\210\001\001\240\001\001" ->>>>>>> updatd review comments for protobuf backward compatibility and moving code to upgrade Util + "\001(\010\022\034\n\024updateCacheFrequency\030\034 \001(\003*A\n\nPTa", + "bleType\022\n\n\006SYSTEM\020\000\022\010\n\004USER\020\001\022\010\n\004VIEW\020\002\022" + + "\t\n\005INDEX\020\003\022\010\n\004JOIN\020\004B@\n(org.apache.phoen" + + "ix.coprocessor.generatedB\014PTableProtosH\001" + + "\210\001\001\240\001\001" }; com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner = new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() {