Perform field selections on lite messages by reading from the wire format#668

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
copybara-service merged 1 commit into from
Apr 25, 2025
Merged

Perform field selections on lite messages by reading from the wire format #668

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions common/src/main/java/dev/cel/common/values/BUILD.bazel
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -179,6 +179,7 @@ java_library(
"//protobuf:cel_lite_descriptor",
"@maven//:com_google_errorprone_error_prone_annotations",
"@maven//:com_google_guava_guava",
"@maven//:com_google_protobuf_protobuf_java",
"@maven//:org_jspecify_jspecify",
"@maven_android//:com_google_protobuf_protobuf_javalite",
],
Expand Down
283 changes: 283 additions & 0 deletions common/src/main/java/dev/cel/common/values/ProtoLiteCelValueConverter.java
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -16,12 +16,31 @@

import static com.google.common.base.Preconditions.checkNotNull;

import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Defaults;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.primitives.UnsignedLong;
import com.google.errorprone.annotations.Immutable;
import com.google.protobuf.ByteString;
import com.google.protobuf.CodedInputStream;
import com.google.protobuf.ExtensionRegistryLite;
import com.google.protobuf.MessageLite;
import com.google.protobuf.WireFormat;
import dev.cel.common.annotations.Internal;
import dev.cel.common.internal.CelLiteDescriptorPool;
import dev.cel.common.internal.WellKnownProto;
import dev.cel.protobuf.CelLiteDescriptor.FieldLiteDescriptor;
import dev.cel.protobuf.CelLiteDescriptor.FieldLiteDescriptor.CelFieldValueType;
import dev.cel.protobuf.CelLiteDescriptor.FieldLiteDescriptor.JavaType;
import dev.cel.protobuf.CelLiteDescriptor.MessageLiteDescriptor;
import java.io.IOException;
import java.util.AbstractMap;
import java.util.ArrayList;
import java.util.Collection;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;

/**
* {@code ProtoLiteCelValueConverter} handles bidirectional conversion between native Java and
Expand All@@ -43,6 +62,270 @@ public static ProtoLiteCelValueConverter newInstance(
return new ProtoLiteCelValueConverter(celLiteDescriptorPool);
}

private static Object readPrimitiveField(
CodedInputStream inputStream, FieldLiteDescriptor fieldDescriptor) throws IOException {
switch (fieldDescriptor.getProtoFieldType()) {
case SINT32:
return inputStream.readSInt32();
case SINT64:
return inputStream.readSInt64();
case INT32:
case ENUM:
return inputStream.readInt32();
case INT64:
return inputStream.readInt64();
case UINT32:
return UnsignedLong.fromLongBits(inputStream.readUInt32());
case UINT64:
return UnsignedLong.fromLongBits(inputStream.readUInt64());
case BOOL:
return inputStream.readBool();
case FLOAT:
case FIXED32:
case SFIXED32:
return readFixed32BitField(inputStream, fieldDescriptor);
case DOUBLE:
case FIXED64:
case SFIXED64:
return readFixed64BitField(inputStream, fieldDescriptor);
default:
throw new IllegalStateException(
"Unexpected field type: " + fieldDescriptor.getProtoFieldType());
}
}

private static Object readFixed32BitField(
CodedInputStream inputStream, FieldLiteDescriptor fieldDescriptor) throws IOException {
switch (fieldDescriptor.getProtoFieldType()) {
case FLOAT:
return inputStream.readFloat();
case FIXED32:
case SFIXED32:
return inputStream.readRawLittleEndian32();
default:
throw new IllegalStateException(
"Unexpected field type: " + fieldDescriptor.getProtoFieldType());
}
}

private static Object readFixed64BitField(
CodedInputStream inputStream, FieldLiteDescriptor fieldDescriptor) throws IOException {
switch (fieldDescriptor.getProtoFieldType()) {
case DOUBLE:
return inputStream.readDouble();
case FIXED64:
case SFIXED64:
return inputStream.readRawLittleEndian64();
default:
throw new IllegalStateException(
"Unexpected field type: " + fieldDescriptor.getProtoFieldType());
}
}

private Object readLengthDelimitedField(
CodedInputStream inputStream, FieldLiteDescriptor fieldDescriptor) throws IOException {
FieldLiteDescriptor.Type fieldType = fieldDescriptor.getProtoFieldType();

switch (fieldType) {
case BYTES:
return inputStream.readBytes();
case MESSAGE:
MessageLite.Builder builder =
getDefaultMessageBuilder(fieldDescriptor.getFieldProtoTypeName());

inputStream.readMessage(builder, ExtensionRegistryLite.getEmptyRegistry());
return builder.build();
case STRING:
return inputStream.readStringRequireUtf8();
default:
throw new IllegalStateException("Unexpected field type: " + fieldType);
}
}

private MessageLite.Builder getDefaultMessageBuilder(String protoTypeName) {
return descriptorPool.getDescriptorOrThrow(protoTypeName).newMessageBuilder();
}

CelValue getDefaultCelValue(String protoTypeName, String fieldName) {
MessageLiteDescriptor messageDescriptor = descriptorPool.getDescriptorOrThrow(protoTypeName);
FieldLiteDescriptor fieldDescriptor = messageDescriptor.getByFieldNameOrThrow(fieldName);

Object defaultValue = getDefaultValue(fieldDescriptor);
if (defaultValue instanceof MessageLite) {
return fromProtoMessageToCelValue(
fieldDescriptor.getFieldProtoTypeName(), (MessageLite) defaultValue);
} else {
return fromJavaObjectToCelValue(getDefaultValue(fieldDescriptor));
}
}

private Object getDefaultValue(FieldLiteDescriptor fieldDescriptor) {
FieldLiteDescriptor.CelFieldValueType celFieldValueType =
fieldDescriptor.getCelFieldValueType();
switch (celFieldValueType) {
case LIST:
return ImmutableList.of();
case MAP:
return ImmutableMap.of();
case SCALAR:
return getScalarDefaultValue(fieldDescriptor);
}
throw new IllegalStateException("Unexpected cel field value type: " + celFieldValueType);
}

private Object getScalarDefaultValue(FieldLiteDescriptor fieldDescriptor) {
JavaType type = fieldDescriptor.getJavaType();
switch (type) {
case INT:
return fieldDescriptor.getProtoFieldType().equals(FieldLiteDescriptor.Type.UINT32)
? UnsignedLong.ZERO
: Defaults.defaultValue(long.class);
case LONG:
return fieldDescriptor.getProtoFieldType().equals(FieldLiteDescriptor.Type.UINT64)
? UnsignedLong.ZERO
: Defaults.defaultValue(long.class);
case ENUM:
return Defaults.defaultValue(long.class);
case FLOAT:
return Defaults.defaultValue(float.class);
case DOUBLE:
return Defaults.defaultValue(double.class);
case BOOLEAN:
return Defaults.defaultValue(boolean.class);
case STRING:
return "";
case BYTE_STRING:
return ByteString.EMPTY;
case MESSAGE:
if (WellKnownProto.isWrapperType(fieldDescriptor.getFieldProtoTypeName())) {
return NullValue.NULL_VALUE;
}

return getDefaultMessageBuilder(fieldDescriptor.getFieldProtoTypeName()).build();
}
throw new IllegalStateException("Unexpected java type: " + type);
}

private ImmutableList<Object> readPackedRepeatedFields(
CodedInputStream inputStream, FieldLiteDescriptor fieldDescriptor) throws IOException {
int length = inputStream.readInt32();
int oldLimit = inputStream.pushLimit(length);
ImmutableList.Builder<Object> builder = ImmutableList.builder();
while (inputStream.getBytesUntilLimit() > 0) {
builder.add(readPrimitiveField(inputStream, fieldDescriptor));
}
inputStream.popLimit(oldLimit);
return builder.build();
}

private Map.Entry<Object, Object> readSingleMapEntry(
CodedInputStream inputStream, FieldLiteDescriptor fieldDescriptor) throws IOException {
ImmutableMap<String, Object> singleMapEntry =
readAllFields(inputStream.readByteArray(), fieldDescriptor.getFieldProtoTypeName());
Object key = checkNotNull(singleMapEntry.get("key"));
Object value = checkNotNull(singleMapEntry.get("value"));

return new AbstractMap.SimpleEntry<>(key, value);
}

@VisibleForTesting
ImmutableMap<String, Object> readAllFields(byte[] bytes, String protoTypeName)
throws IOException {
// TODO: Handle unknown fields by collecting them into a separate map.
MessageLiteDescriptor messageDescriptor = descriptorPool.getDescriptorOrThrow(protoTypeName);
CodedInputStream inputStream = CodedInputStream.newInstance(bytes);

ImmutableMap.Builder<String, Object> fieldValues = ImmutableMap.builder();
Map<Integer, List<Object>> repeatedFieldValues = new LinkedHashMap<>();
Map<Integer, Map<Object, Object>> mapFieldValues = new LinkedHashMap<>();
for (int tag = inputStream.readTag(); tag != 0; tag = inputStream.readTag()) {
int tagWireType = WireFormat.getTagWireType(tag);
int fieldNumber = WireFormat.getTagFieldNumber(tag);
FieldLiteDescriptor fieldDescriptor = messageDescriptor.getByFieldNumberOrThrow(fieldNumber);

Object payload;
switch (tagWireType) {
case WireFormat.WIRETYPE_VARINT:
payload = readPrimitiveField(inputStream, fieldDescriptor);
break;
case WireFormat.WIRETYPE_FIXED32:
payload = readFixed32BitField(inputStream, fieldDescriptor);
break;
case WireFormat.WIRETYPE_FIXED64:
payload = readFixed64BitField(inputStream, fieldDescriptor);
break;
case WireFormat.WIRETYPE_LENGTH_DELIMITED:
CelFieldValueType celFieldValueType = fieldDescriptor.getCelFieldValueType();
switch (celFieldValueType) {
case LIST:
if (fieldDescriptor.getIsPacked()) {
payload = readPackedRepeatedFields(inputStream, fieldDescriptor);
} else {
FieldLiteDescriptor.Type protoFieldType = fieldDescriptor.getProtoFieldType();
boolean isLenDelimited =
protoFieldType.equals(FieldLiteDescriptor.Type.MESSAGE)
|| protoFieldType.equals(FieldLiteDescriptor.Type.STRING)
|| protoFieldType.equals(FieldLiteDescriptor.Type.BYTES);
if (!isLenDelimited) {
throw new IllegalStateException(
"Unexpected field type encountered for LEN-Delimited record: "
+ protoFieldType);
}

payload = readLengthDelimitedField(inputStream, fieldDescriptor);
}
break;
case MAP:
Map<Object, Object> fieldMap =
mapFieldValues.computeIfAbsent(fieldNumber, (unused) -> new LinkedHashMap<>());
Map.Entry<Object, Object> mapEntry = readSingleMapEntry(inputStream, fieldDescriptor);
fieldMap.put(mapEntry.getKey(), mapEntry.getValue());
payload = fieldMap;
break;
default:
payload = readLengthDelimitedField(inputStream, fieldDescriptor);
break;
}
break;
case WireFormat.WIRETYPE_START_GROUP:
case WireFormat.WIRETYPE_END_GROUP:
// TODO: Support groups
throw new UnsupportedOperationException("Groups are not supported");
default:
throw new IllegalArgumentException("Unexpected wire type: " + tagWireType);
}

if (fieldDescriptor.getCelFieldValueType().equals(CelFieldValueType.LIST)) {
String fieldName = fieldDescriptor.getFieldName();
List<Object> repeatedValues =
repeatedFieldValues.computeIfAbsent(
fieldNumber,
(unused) -> {
List<Object> newList = new ArrayList<>();
fieldValues.put(fieldName, newList);
return newList;
});

if (payload instanceof Collection) {
repeatedValues.addAll((Collection<?>) payload);
} else {
repeatedValues.add(payload);
}
} else {
fieldValues.put(fieldDescriptor.getFieldName(), payload);
}
}

// Protobuf encoding follows a "last one wins" semantics. This means for duplicated fields,
// we accept the last value encountered.
return fieldValues.buildKeepingLast();
}

ImmutableMap<String, Object> readAllFields(MessageLite msg, String protoTypeName)
throws IOException {
return readAllFields(msg.toByteArray(), protoTypeName);
}

@Override
public CelValue fromProtoMessageToCelValue(String protoTypeName, MessageLite msg) {
checkNotNull(msg);
Expand Down
20 changes: 18 additions & 2 deletions common/src/main/java/dev/cel/common/values/ProtoMessageLiteValue.java
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -15,11 +15,14 @@
package dev.cel.common.values;

import com.google.auto.value.AutoValue;
import com.google.auto.value.extension.memoized.Memoized;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableMap;
import com.google.errorprone.annotations.Immutable;
import com.google.protobuf.MessageLite;
import dev.cel.common.types.CelType;
import dev.cel.common.types.StructTypeReference;
import java.io.IOException;
import java.util.Optional;

/**
Expand All@@ -42,19 +45,32 @@ public abstract class ProtoMessageLiteValue extends StructValue<StringValue> {

abstract ProtoLiteCelValueConverter protoLiteCelValueConverter();

@Memoized
ImmutableMap<String, Object> fieldValues() {
try {
return protoLiteCelValueConverter().readAllFields(value(), celType().name());
} catch (IOException e) {
throw new IllegalStateException("Unable to read message fields for " + celType().name(), e);
}
}

@Override
public boolean isZeroValue() {
return value().getDefaultInstanceForType().equals(value());
}

@Override
public CelValue select(StringValue field) {
throw new UnsupportedOperationException("Not implemented yet");
return find(field)
.orElseGet(
() -> protoLiteCelValueConverter().getDefaultCelValue(celType().name(), field.value()));
}

@Override
public Optional<CelValue> find(StringValue field) {
throw new UnsupportedOperationException("Not implemented yet");
Object fieldValue = fieldValues().get(field.value());
return Optional.ofNullable(fieldValue)
.map(value -> protoLiteCelValueConverter().fromJavaObjectToCelValue(fieldValue));
}

public static ProtoMessageLiteValue create(
Expand Down
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Add copy buttons to all 
 blocks\n(function() {\n function addCopyButtons() {\n document.querySelectorAll('pre code').forEach(function(codeBlock) {\n if (codeBlock.parentElement.hasAttribute('data-copy-added')) return;\n codeBlock.parentElement.setAttribute('data-copy-added', 'true');\n \n var btn = document.createElement('button');\n btn.textContent = 'Copy';\n btn.style.cssText = 'position:absolute;top:4px;right:4px;padding:2px 8px;font-size:11px;background:#4ecdc4;border:none;border-radius:4px;color:#1a1a2e;cursor:pointer;opacity:0.7;transition:opacity 0.2s;';\n btn.onmouseover = function() { this.style.opacity = '1'; };\n btn.onmouseout = function() { this.style.opacity = '0.7'; };\n btn.onclick = function() {\n navigator.clipboard.writeText(codeBlock.textContent).then(function() {\n btn.textContent = 'Copied!';\n setTimeout(function() { btn.textContent = 'Copy'; }, 1500);\n });\n };\n codeBlock.parentElement.style.position = 'relative';\n codeBlock.parentElement.appendChild(btn);\n });\n }\n \n addCopyButtons();\n \n // Re-run on dynamic content\n var observer = new MutationObserver(addCopyButtons);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Add Copy Buttons to Code Blocks");
}
} catch(__e) { console.warn('[Userscript:Add Copy Buttons to Code Blocks]', __e); }
})();
(function(){
try {
var __m = "github.com";
var __re = new RegExp('^' + "github\\.com" + '

Skip to content
Perform field selections on lite messages by reading from the wire format#668

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

copybara-service
merged 1 commit into

from

Apr 25, 2025
 Merged


Perform field selections on lite messages by reading from the wire format
#668
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


1 change: 1 addition & 0 deletions
common/src/main/java/dev/cel/common/values/BUILD.bazel

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -179,6 +179,7 @@ java_library(
 "//protobuf:cel_lite_descriptor",
 "@maven//:com_google_errorprone_error_prone_annotations",
 "@maven//:com_google_guava_guava",
 "@maven//:com_google_protobuf_protobuf_java",
 "@maven//:org_jspecify_jspecify",
 "@maven_android//:com_google_protobuf_protobuf_javalite",
 ],
Expand Down

283 changes: 283 additions & 0 deletions
common/src/main/java/dev/cel/common/values/ProtoLiteCelValueConverter.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -16,12 +16,31 @@

import static com.google.common.base.Preconditions.checkNotNull;

import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Defaults;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.primitives.UnsignedLong;
import com.google.errorprone.annotations.Immutable;
import com.google.protobuf.ByteString;
import com.google.protobuf.CodedInputStream;
import com.google.protobuf.ExtensionRegistryLite;
import com.google.protobuf.MessageLite;
import com.google.protobuf.WireFormat;
import dev.cel.common.annotations.Internal;
import dev.cel.common.internal.CelLiteDescriptorPool;
import dev.cel.common.internal.WellKnownProto;
import dev.cel.protobuf.CelLiteDescriptor.FieldLiteDescriptor;
import dev.cel.protobuf.CelLiteDescriptor.FieldLiteDescriptor.CelFieldValueType;
import dev.cel.protobuf.CelLiteDescriptor.FieldLiteDescriptor.JavaType;
import dev.cel.protobuf.CelLiteDescriptor.MessageLiteDescriptor;
import java.io.IOException;
import java.util.AbstractMap;
import java.util.ArrayList;
import java.util.Collection;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;

/**
 * {@code ProtoLiteCelValueConverter} handles bidirectional conversion between native Java and
Expand All@@ -43,6 +62,270 @@ public static ProtoLiteCelValueConverter newInstance(
 return new ProtoLiteCelValueConverter(celLiteDescriptorPool);
 }

 private static Object readPrimitiveField(
 CodedInputStream inputStream, FieldLiteDescriptor fieldDescriptor) throws IOException {
 switch (fieldDescriptor.getProtoFieldType()) {
 case SINT32:
 return inputStream.readSInt32();
 case SINT64:
 return inputStream.readSInt64();
 case INT32:
 case ENUM:
 return inputStream.readInt32();
 case INT64:
 return inputStream.readInt64();
 case UINT32:
 return UnsignedLong.fromLongBits(inputStream.readUInt32());
 case UINT64:
 return UnsignedLong.fromLongBits(inputStream.readUInt64());
 case BOOL:
 return inputStream.readBool();
 case FLOAT:
 case FIXED32:
 case SFIXED32:
 return readFixed32BitField(inputStream, fieldDescriptor);
 case DOUBLE:
 case FIXED64:
 case SFIXED64:
 return readFixed64BitField(inputStream, fieldDescriptor);
 default:
 throw new IllegalStateException(
 "Unexpected field type: " + fieldDescriptor.getProtoFieldType());
 }
 }

 private static Object readFixed32BitField(
 CodedInputStream inputStream, FieldLiteDescriptor fieldDescriptor) throws IOException {
 switch (fieldDescriptor.getProtoFieldType()) {
 case FLOAT:
 return inputStream.readFloat();
 case FIXED32:
 case SFIXED32:
 return inputStream.readRawLittleEndian32();
 default:
 throw new IllegalStateException(
 "Unexpected field type: " + fieldDescriptor.getProtoFieldType());
 }
 }

 private static Object readFixed64BitField(
 CodedInputStream inputStream, FieldLiteDescriptor fieldDescriptor) throws IOException {
 switch (fieldDescriptor.getProtoFieldType()) {
 case DOUBLE:
 return inputStream.readDouble();
 case FIXED64:
 case SFIXED64:
 return inputStream.readRawLittleEndian64();
 default:
 throw new IllegalStateException(
 "Unexpected field type: " + fieldDescriptor.getProtoFieldType());
 }
 }

 private Object readLengthDelimitedField(
 CodedInputStream inputStream, FieldLiteDescriptor fieldDescriptor) throws IOException {
 FieldLiteDescriptor.Type fieldType = fieldDescriptor.getProtoFieldType();

 switch (fieldType) {
 case BYTES:
 return inputStream.readBytes();
 case MESSAGE:
 MessageLite.Builder builder =
 getDefaultMessageBuilder(fieldDescriptor.getFieldProtoTypeName());

 inputStream.readMessage(builder, ExtensionRegistryLite.getEmptyRegistry());
 return builder.build();
 case STRING:
 return inputStream.readStringRequireUtf8();
 default:
 throw new IllegalStateException("Unexpected field type: " + fieldType);
 }
 }

 private MessageLite.Builder getDefaultMessageBuilder(String protoTypeName) {
 return descriptorPool.getDescriptorOrThrow(protoTypeName).newMessageBuilder();
 }

 CelValue getDefaultCelValue(String protoTypeName, String fieldName) {
 MessageLiteDescriptor messageDescriptor = descriptorPool.getDescriptorOrThrow(protoTypeName);
 FieldLiteDescriptor fieldDescriptor = messageDescriptor.getByFieldNameOrThrow(fieldName);

 Object defaultValue = getDefaultValue(fieldDescriptor);
 if (defaultValue instanceof MessageLite) {
 return fromProtoMessageToCelValue(
 fieldDescriptor.getFieldProtoTypeName(), (MessageLite) defaultValue);
 } else {
 return fromJavaObjectToCelValue(getDefaultValue(fieldDescriptor));
 }
 }

 private Object getDefaultValue(FieldLiteDescriptor fieldDescriptor) {
 FieldLiteDescriptor.CelFieldValueType celFieldValueType =
 fieldDescriptor.getCelFieldValueType();
 switch (celFieldValueType) {
 case LIST:
 return ImmutableList.of();
 case MAP:
 return ImmutableMap.of();
 case SCALAR:
 return getScalarDefaultValue(fieldDescriptor);
 }
 throw new IllegalStateException("Unexpected cel field value type: " + celFieldValueType);
 }

 private Object getScalarDefaultValue(FieldLiteDescriptor fieldDescriptor) {
 JavaType type = fieldDescriptor.getJavaType();
 switch (type) {
 case INT:
 return fieldDescriptor.getProtoFieldType().equals(FieldLiteDescriptor.Type.UINT32)
 ? UnsignedLong.ZERO
 : Defaults.defaultValue(long.class);
 case LONG:
 return fieldDescriptor.getProtoFieldType().equals(FieldLiteDescriptor.Type.UINT64)
 ? UnsignedLong.ZERO
 : Defaults.defaultValue(long.class);
 case ENUM:
 return Defaults.defaultValue(long.class);
 case FLOAT:
 return Defaults.defaultValue(float.class);
 case DOUBLE:
 return Defaults.defaultValue(double.class);
 case BOOLEAN:
 return Defaults.defaultValue(boolean.class);
 case STRING:
 return "";
 case BYTE_STRING:
 return ByteString.EMPTY;
 case MESSAGE:
 if (WellKnownProto.isWrapperType(fieldDescriptor.getFieldProtoTypeName())) {
 return NullValue.NULL_VALUE;
 }

 return getDefaultMessageBuilder(fieldDescriptor.getFieldProtoTypeName()).build();
 }
 throw new IllegalStateException("Unexpected java type: " + type);
 }

 private ImmutableList<Object> readPackedRepeatedFields(
 CodedInputStream inputStream, FieldLiteDescriptor fieldDescriptor) throws IOException {
 int length = inputStream.readInt32();
 int oldLimit = inputStream.pushLimit(length);
 ImmutableList.Builder<Object> builder = ImmutableList.builder();
 while (inputStream.getBytesUntilLimit() > 0) {
 builder.add(readPrimitiveField(inputStream, fieldDescriptor));
 }
 inputStream.popLimit(oldLimit);
 return builder.build();
 }

 private Map.Entry<Object, Object> readSingleMapEntry(
 CodedInputStream inputStream, FieldLiteDescriptor fieldDescriptor) throws IOException {
 ImmutableMap<String, Object> singleMapEntry =
 readAllFields(inputStream.readByteArray(), fieldDescriptor.getFieldProtoTypeName());
 Object key = checkNotNull(singleMapEntry.get("key"));
 Object value = checkNotNull(singleMapEntry.get("value"));

 return new AbstractMap.SimpleEntry<>(key, value);
 }

 @VisibleForTesting
 ImmutableMap<String, Object> readAllFields(byte[] bytes, String protoTypeName)
 throws IOException {
 // TODO: Handle unknown fields by collecting them into a separate map.
 MessageLiteDescriptor messageDescriptor = descriptorPool.getDescriptorOrThrow(protoTypeName);
 CodedInputStream inputStream = CodedInputStream.newInstance(bytes);

 ImmutableMap.Builder<String, Object> fieldValues = ImmutableMap.builder();
 Map<Integer, List<Object>> repeatedFieldValues = new LinkedHashMap<>();
 Map<Integer, Map<Object, Object>> mapFieldValues = new LinkedHashMap<>();
 for (int tag = inputStream.readTag(); tag != 0; tag = inputStream.readTag()) {
 int tagWireType = WireFormat.getTagWireType(tag);
 int fieldNumber = WireFormat.getTagFieldNumber(tag);
 FieldLiteDescriptor fieldDescriptor = messageDescriptor.getByFieldNumberOrThrow(fieldNumber);

 Object payload;
 switch (tagWireType) {
 case WireFormat.WIRETYPE_VARINT:
 payload = readPrimitiveField(inputStream, fieldDescriptor);
 break;
 case WireFormat.WIRETYPE_FIXED32:
 payload = readFixed32BitField(inputStream, fieldDescriptor);
 break;
 case WireFormat.WIRETYPE_FIXED64:
 payload = readFixed64BitField(inputStream, fieldDescriptor);
 break;
 case WireFormat.WIRETYPE_LENGTH_DELIMITED:
 CelFieldValueType celFieldValueType = fieldDescriptor.getCelFieldValueType();
 switch (celFieldValueType) {
 case LIST:
 if (fieldDescriptor.getIsPacked()) {
 payload = readPackedRepeatedFields(inputStream, fieldDescriptor);
 } else {
 FieldLiteDescriptor.Type protoFieldType = fieldDescriptor.getProtoFieldType();
 boolean isLenDelimited =
 protoFieldType.equals(FieldLiteDescriptor.Type.MESSAGE)
 || protoFieldType.equals(FieldLiteDescriptor.Type.STRING)
 || protoFieldType.equals(FieldLiteDescriptor.Type.BYTES);
 if (!isLenDelimited) {
 throw new IllegalStateException(
 "Unexpected field type encountered for LEN-Delimited record: "
 + protoFieldType);
 }

 payload = readLengthDelimitedField(inputStream, fieldDescriptor);
 }
 break;
 case MAP:
 Map<Object, Object> fieldMap =
 mapFieldValues.computeIfAbsent(fieldNumber, (unused) -> new LinkedHashMap<>());
 Map.Entry<Object, Object> mapEntry = readSingleMapEntry(inputStream, fieldDescriptor);
 fieldMap.put(mapEntry.getKey(), mapEntry.getValue());
 payload = fieldMap;
 break;
 default:
 payload = readLengthDelimitedField(inputStream, fieldDescriptor);
 break;
 }
 break;
 case WireFormat.WIRETYPE_START_GROUP:
 case WireFormat.WIRETYPE_END_GROUP:
 // TODO: Support groups
 throw new UnsupportedOperationException("Groups are not supported");
 default:
 throw new IllegalArgumentException("Unexpected wire type: " + tagWireType);
 }

 if (fieldDescriptor.getCelFieldValueType().equals(CelFieldValueType.LIST)) {
 String fieldName = fieldDescriptor.getFieldName();
 List<Object> repeatedValues =
 repeatedFieldValues.computeIfAbsent(
 fieldNumber,
 (unused) -> {
 List<Object> newList = new ArrayList<>();
 fieldValues.put(fieldName, newList);
 return newList;
 });

 if (payload instanceof Collection) {
 repeatedValues.addAll((Collection<?>) payload);
 } else {
 repeatedValues.add(payload);
 }
 } else {
 fieldValues.put(fieldDescriptor.getFieldName(), payload);
 }
 }

 // Protobuf encoding follows a "last one wins" semantics. This means for duplicated fields,
 // we accept the last value encountered.
 return fieldValues.buildKeepingLast();
 }

 ImmutableMap<String, Object> readAllFields(MessageLite msg, String protoTypeName)
 throws IOException {
 return readAllFields(msg.toByteArray(), protoTypeName);
 }

 @Override
 public CelValue fromProtoMessageToCelValue(String protoTypeName, MessageLite msg) {
 checkNotNull(msg);
Expand Down

20 changes: 18 additions & 2 deletions
common/src/main/java/dev/cel/common/values/ProtoMessageLiteValue.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -15,11 +15,14 @@
package dev.cel.common.values;

import com.google.auto.value.AutoValue;
import com.google.auto.value.extension.memoized.Memoized;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableMap;
import com.google.errorprone.annotations.Immutable;
import com.google.protobuf.MessageLite;
import dev.cel.common.types.CelType;
import dev.cel.common.types.StructTypeReference;
import java.io.IOException;
import java.util.Optional;

/**
Expand All@@ -42,19 +45,32 @@ public abstract class ProtoMessageLiteValue extends StructValue<StringValue> {

 abstract ProtoLiteCelValueConverter protoLiteCelValueConverter();

 @Memoized
 ImmutableMap<String, Object> fieldValues() {
 try {
 return protoLiteCelValueConverter().readAllFields(value(), celType().name());
 } catch (IOException e) {
 throw new IllegalStateException("Unable to read message fields for " + celType().name(), e);
 }
 }

 @Override
 public boolean isZeroValue() {
 return value().getDefaultInstanceForType().equals(value());
 }

 @Override
 public CelValue select(StringValue field) {
 throw new UnsupportedOperationException("Not implemented yet");
 return find(field)
 .orElseGet(
 () -> protoLiteCelValueConverter().getDefaultCelValue(celType().name(), field.value()));
 }

 @Override
 public Optional<CelValue> find(StringValue field) {
 throw new UnsupportedOperationException("Not implemented yet");
 Object fieldValue = fieldValues().get(field.value());
 return Optional.ofNullable(fieldValue)
 .map(value -> protoLiteCelValueConverter().fromJavaObjectToCelValue(fieldValue));
 }

 public static ProtoMessageLiteValue create(
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Force GitHub README to respect dark mode\n(function() {\n var style = document.createElement('style');\n style.textContent = '\n .markdown-body {\n color-scheme: dark light;\n }\n .markdown-body pre { background: #161b22 !important; }\n .markdown-body code { background: rgba(110, 118, 129, 0.4) !important; }\n .markdown-body table th, .markdown-body table td { border-color: #30363d !important; }\n .markdown-body img { background: #0d1117; }\n .markdown-body blockquote { border-left-color: #8b949e; }\n .markdown-body hr { border-color: #30363d; }\n ';\n document.head.appendChild(style);\n})();", "GitHub Dark Mode README Fix");
}
} catch(__e) { console.warn('[Userscript:GitHub Dark Mode README Fix]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
Perform field selections on lite messages by reading from the wire format#668

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

copybara-service
merged 1 commit into

from

Apr 25, 2025
 Merged


Perform field selections on lite messages by reading from the wire format
#668
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


1 change: 1 addition & 0 deletions
common/src/main/java/dev/cel/common/values/BUILD.bazel

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -179,6 +179,7 @@ java_library(
 "//protobuf:cel_lite_descriptor",
 "@maven//:com_google_errorprone_error_prone_annotations",
 "@maven//:com_google_guava_guava",
 "@maven//:com_google_protobuf_protobuf_java",
 "@maven//:org_jspecify_jspecify",
 "@maven_android//:com_google_protobuf_protobuf_javalite",
 ],
Expand Down

283 changes: 283 additions & 0 deletions
common/src/main/java/dev/cel/common/values/ProtoLiteCelValueConverter.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -16,12 +16,31 @@

import static com.google.common.base.Preconditions.checkNotNull;

import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Defaults;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.primitives.UnsignedLong;
import com.google.errorprone.annotations.Immutable;
import com.google.protobuf.ByteString;
import com.google.protobuf.CodedInputStream;
import com.google.protobuf.ExtensionRegistryLite;
import com.google.protobuf.MessageLite;
import com.google.protobuf.WireFormat;
import dev.cel.common.annotations.Internal;
import dev.cel.common.internal.CelLiteDescriptorPool;
import dev.cel.common.internal.WellKnownProto;
import dev.cel.protobuf.CelLiteDescriptor.FieldLiteDescriptor;
import dev.cel.protobuf.CelLiteDescriptor.FieldLiteDescriptor.CelFieldValueType;
import dev.cel.protobuf.CelLiteDescriptor.FieldLiteDescriptor.JavaType;
import dev.cel.protobuf.CelLiteDescriptor.MessageLiteDescriptor;
import java.io.IOException;
import java.util.AbstractMap;
import java.util.ArrayList;
import java.util.Collection;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;

/**
 * {@code ProtoLiteCelValueConverter} handles bidirectional conversion between native Java and
Expand All@@ -43,6 +62,270 @@ public static ProtoLiteCelValueConverter newInstance(
 return new ProtoLiteCelValueConverter(celLiteDescriptorPool);
 }

 private static Object readPrimitiveField(
 CodedInputStream inputStream, FieldLiteDescriptor fieldDescriptor) throws IOException {
 switch (fieldDescriptor.getProtoFieldType()) {
 case SINT32:
 return inputStream.readSInt32();
 case SINT64:
 return inputStream.readSInt64();
 case INT32:
 case ENUM:
 return inputStream.readInt32();
 case INT64:
 return inputStream.readInt64();
 case UINT32:
 return UnsignedLong.fromLongBits(inputStream.readUInt32());
 case UINT64:
 return UnsignedLong.fromLongBits(inputStream.readUInt64());
 case BOOL:
 return inputStream.readBool();
 case FLOAT:
 case FIXED32:
 case SFIXED32:
 return readFixed32BitField(inputStream, fieldDescriptor);
 case DOUBLE:
 case FIXED64:
 case SFIXED64:
 return readFixed64BitField(inputStream, fieldDescriptor);
 default:
 throw new IllegalStateException(
 "Unexpected field type: " + fieldDescriptor.getProtoFieldType());
 }
 }

 private static Object readFixed32BitField(
 CodedInputStream inputStream, FieldLiteDescriptor fieldDescriptor) throws IOException {
 switch (fieldDescriptor.getProtoFieldType()) {
 case FLOAT:
 return inputStream.readFloat();
 case FIXED32:
 case SFIXED32:
 return inputStream.readRawLittleEndian32();
 default:
 throw new IllegalStateException(
 "Unexpected field type: " + fieldDescriptor.getProtoFieldType());
 }
 }

 private static Object readFixed64BitField(
 CodedInputStream inputStream, FieldLiteDescriptor fieldDescriptor) throws IOException {
 switch (fieldDescriptor.getProtoFieldType()) {
 case DOUBLE:
 return inputStream.readDouble();
 case FIXED64:
 case SFIXED64:
 return inputStream.readRawLittleEndian64();
 default:
 throw new IllegalStateException(
 "Unexpected field type: " + fieldDescriptor.getProtoFieldType());
 }
 }

 private Object readLengthDelimitedField(
 CodedInputStream inputStream, FieldLiteDescriptor fieldDescriptor) throws IOException {
 FieldLiteDescriptor.Type fieldType = fieldDescriptor.getProtoFieldType();

 switch (fieldType) {
 case BYTES:
 return inputStream.readBytes();
 case MESSAGE:
 MessageLite.Builder builder =
 getDefaultMessageBuilder(fieldDescriptor.getFieldProtoTypeName());

 inputStream.readMessage(builder, ExtensionRegistryLite.getEmptyRegistry());
 return builder.build();
 case STRING:
 return inputStream.readStringRequireUtf8();
 default:
 throw new IllegalStateException("Unexpected field type: " + fieldType);
 }
 }

 private MessageLite.Builder getDefaultMessageBuilder(String protoTypeName) {
 return descriptorPool.getDescriptorOrThrow(protoTypeName).newMessageBuilder();
 }

 CelValue getDefaultCelValue(String protoTypeName, String fieldName) {
 MessageLiteDescriptor messageDescriptor = descriptorPool.getDescriptorOrThrow(protoTypeName);
 FieldLiteDescriptor fieldDescriptor = messageDescriptor.getByFieldNameOrThrow(fieldName);

 Object defaultValue = getDefaultValue(fieldDescriptor);
 if (defaultValue instanceof MessageLite) {
 return fromProtoMessageToCelValue(
 fieldDescriptor.getFieldProtoTypeName(), (MessageLite) defaultValue);
 } else {
 return fromJavaObjectToCelValue(getDefaultValue(fieldDescriptor));
 }
 }

 private Object getDefaultValue(FieldLiteDescriptor fieldDescriptor) {
 FieldLiteDescriptor.CelFieldValueType celFieldValueType =
 fieldDescriptor.getCelFieldValueType();
 switch (celFieldValueType) {
 case LIST:
 return ImmutableList.of();
 case MAP:
 return ImmutableMap.of();
 case SCALAR:
 return getScalarDefaultValue(fieldDescriptor);
 }
 throw new IllegalStateException("Unexpected cel field value type: " + celFieldValueType);
 }

 private Object getScalarDefaultValue(FieldLiteDescriptor fieldDescriptor) {
 JavaType type = fieldDescriptor.getJavaType();
 switch (type) {
 case INT:
 return fieldDescriptor.getProtoFieldType().equals(FieldLiteDescriptor.Type.UINT32)
 ? UnsignedLong.ZERO
 : Defaults.defaultValue(long.class);
 case LONG:
 return fieldDescriptor.getProtoFieldType().equals(FieldLiteDescriptor.Type.UINT64)
 ? UnsignedLong.ZERO
 : Defaults.defaultValue(long.class);
 case ENUM:
 return Defaults.defaultValue(long.class);
 case FLOAT:
 return Defaults.defaultValue(float.class);
 case DOUBLE:
 return Defaults.defaultValue(double.class);
 case BOOLEAN:
 return Defaults.defaultValue(boolean.class);
 case STRING:
 return "";
 case BYTE_STRING:
 return ByteString.EMPTY;
 case MESSAGE:
 if (WellKnownProto.isWrapperType(fieldDescriptor.getFieldProtoTypeName())) {
 return NullValue.NULL_VALUE;
 }

 return getDefaultMessageBuilder(fieldDescriptor.getFieldProtoTypeName()).build();
 }
 throw new IllegalStateException("Unexpected java type: " + type);
 }

 private ImmutableList<Object> readPackedRepeatedFields(
 CodedInputStream inputStream, FieldLiteDescriptor fieldDescriptor) throws IOException {
 int length = inputStream.readInt32();
 int oldLimit = inputStream.pushLimit(length);
 ImmutableList.Builder<Object> builder = ImmutableList.builder();
 while (inputStream.getBytesUntilLimit() > 0) {
 builder.add(readPrimitiveField(inputStream, fieldDescriptor));
 }
 inputStream.popLimit(oldLimit);
 return builder.build();
 }

 private Map.Entry<Object, Object> readSingleMapEntry(
 CodedInputStream inputStream, FieldLiteDescriptor fieldDescriptor) throws IOException {
 ImmutableMap<String, Object> singleMapEntry =
 readAllFields(inputStream.readByteArray(), fieldDescriptor.getFieldProtoTypeName());
 Object key = checkNotNull(singleMapEntry.get("key"));
 Object value = checkNotNull(singleMapEntry.get("value"));

 return new AbstractMap.SimpleEntry<>(key, value);
 }

 @VisibleForTesting
 ImmutableMap<String, Object> readAllFields(byte[] bytes, String protoTypeName)
 throws IOException {
 // TODO: Handle unknown fields by collecting them into a separate map.
 MessageLiteDescriptor messageDescriptor = descriptorPool.getDescriptorOrThrow(protoTypeName);
 CodedInputStream inputStream = CodedInputStream.newInstance(bytes);

 ImmutableMap.Builder<String, Object> fieldValues = ImmutableMap.builder();
 Map<Integer, List<Object>> repeatedFieldValues = new LinkedHashMap<>();
 Map<Integer, Map<Object, Object>> mapFieldValues = new LinkedHashMap<>();
 for (int tag = inputStream.readTag(); tag != 0; tag = inputStream.readTag()) {
 int tagWireType = WireFormat.getTagWireType(tag);
 int fieldNumber = WireFormat.getTagFieldNumber(tag);
 FieldLiteDescriptor fieldDescriptor = messageDescriptor.getByFieldNumberOrThrow(fieldNumber);

 Object payload;
 switch (tagWireType) {
 case WireFormat.WIRETYPE_VARINT:
 payload = readPrimitiveField(inputStream, fieldDescriptor);
 break;
 case WireFormat.WIRETYPE_FIXED32:
 payload = readFixed32BitField(inputStream, fieldDescriptor);
 break;
 case WireFormat.WIRETYPE_FIXED64:
 payload = readFixed64BitField(inputStream, fieldDescriptor);
 break;
 case WireFormat.WIRETYPE_LENGTH_DELIMITED:
 CelFieldValueType celFieldValueType = fieldDescriptor.getCelFieldValueType();
 switch (celFieldValueType) {
 case LIST:
 if (fieldDescriptor.getIsPacked()) {
 payload = readPackedRepeatedFields(inputStream, fieldDescriptor);
 } else {
 FieldLiteDescriptor.Type protoFieldType = fieldDescriptor.getProtoFieldType();
 boolean isLenDelimited =
 protoFieldType.equals(FieldLiteDescriptor.Type.MESSAGE)
 || protoFieldType.equals(FieldLiteDescriptor.Type.STRING)
 || protoFieldType.equals(FieldLiteDescriptor.Type.BYTES);
 if (!isLenDelimited) {
 throw new IllegalStateException(
 "Unexpected field type encountered for LEN-Delimited record: "
 + protoFieldType);
 }

 payload = readLengthDelimitedField(inputStream, fieldDescriptor);
 }
 break;
 case MAP:
 Map<Object, Object> fieldMap =
 mapFieldValues.computeIfAbsent(fieldNumber, (unused) -> new LinkedHashMap<>());
 Map.Entry<Object, Object> mapEntry = readSingleMapEntry(inputStream, fieldDescriptor);
 fieldMap.put(mapEntry.getKey(), mapEntry.getValue());
 payload = fieldMap;
 break;
 default:
 payload = readLengthDelimitedField(inputStream, fieldDescriptor);
 break;
 }
 break;
 case WireFormat.WIRETYPE_START_GROUP:
 case WireFormat.WIRETYPE_END_GROUP:
 // TODO: Support groups
 throw new UnsupportedOperationException("Groups are not supported");
 default:
 throw new IllegalArgumentException("Unexpected wire type: " + tagWireType);
 }

 if (fieldDescriptor.getCelFieldValueType().equals(CelFieldValueType.LIST)) {
 String fieldName = fieldDescriptor.getFieldName();
 List<Object> repeatedValues =
 repeatedFieldValues.computeIfAbsent(
 fieldNumber,
 (unused) -> {
 List<Object> newList = new ArrayList<>();
 fieldValues.put(fieldName, newList);
 return newList;
 });

 if (payload instanceof Collection) {
 repeatedValues.addAll((Collection<?>) payload);
 } else {
 repeatedValues.add(payload);
 }
 } else {
 fieldValues.put(fieldDescriptor.getFieldName(), payload);
 }
 }

 // Protobuf encoding follows a "last one wins" semantics. This means for duplicated fields,
 // we accept the last value encountered.
 return fieldValues.buildKeepingLast();
 }

 ImmutableMap<String, Object> readAllFields(MessageLite msg, String protoTypeName)
 throws IOException {
 return readAllFields(msg.toByteArray(), protoTypeName);
 }

 @Override
 public CelValue fromProtoMessageToCelValue(String protoTypeName, MessageLite msg) {
 checkNotNull(msg);
Expand Down

20 changes: 18 additions & 2 deletions
common/src/main/java/dev/cel/common/values/ProtoMessageLiteValue.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -15,11 +15,14 @@
package dev.cel.common.values;

import com.google.auto.value.AutoValue;
import com.google.auto.value.extension.memoized.Memoized;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableMap;
import com.google.errorprone.annotations.Immutable;
import com.google.protobuf.MessageLite;
import dev.cel.common.types.CelType;
import dev.cel.common.types.StructTypeReference;
import java.io.IOException;
import java.util.Optional;

/**
Expand All@@ -42,19 +45,32 @@ public abstract class ProtoMessageLiteValue extends StructValue<StringValue> {

 abstract ProtoLiteCelValueConverter protoLiteCelValueConverter();

 @Memoized
 ImmutableMap<String, Object> fieldValues() {
 try {
 return protoLiteCelValueConverter().readAllFields(value(), celType().name());
 } catch (IOException e) {
 throw new IllegalStateException("Unable to read message fields for " + celType().name(), e);
 }
 }

 @Override
 public boolean isZeroValue() {
 return value().getDefaultInstanceForType().equals(value());
 }

 @Override
 public CelValue select(StringValue field) {
 throw new UnsupportedOperationException("Not implemented yet");
 return find(field)
 .orElseGet(
 () -> protoLiteCelValueConverter().getDefaultCelValue(celType().name(), field.value()));
 }

 @Override
 public Optional<CelValue> find(StringValue field) {
 throw new UnsupportedOperationException("Not implemented yet");
 Object fieldValue = fieldValues().get(field.value());
 return Optional.ofNullable(fieldValue)
 .map(value -> protoLiteCelValueConverter().fromJavaObjectToCelValue(fieldValue));
 }

 public static ProtoMessageLiteValue create(
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Highlight search terms from Google/DuckDuckGo/Bing referrer\n(function() {\n var ref = document.referrer;\n var terms = [];\n \n if (ref.includes('google.com') || ref.includes('duckduckgo.com') || ref.includes('bing.com')) {\n var url = new URL(ref);\n var q = url.searchParams.get('q') || url.searchParams.get('p');\n if (q) {\n terms = q.split(/\\s+/).filter(function(t) { return t.length > 2; });\n }\n }\n \n if (terms.length === 0) return;\n \n var style = document.createElement('style');\n style.textContent = '.userscript-highlight { background: #fbbf24; color: #1a1a2e; padding: 1px 3px; border-radius: 2px; }';\n document.head.appendChild(style);\n \n function highlight(node) {\n if (node.nodeType === 3) { // text node\n var text = node.textContent;\n var found = false;\n terms.forEach(function(term) {\n var regex = new RegExp('(' + term.replace(/[.*+?^${}()|[\\]\\\\]/g, '\\\\') + ')', 'gi');\n if (regex.test(text)) {\n found = true;\n var frag = document.createDocumentFragment();\n var parts = text.split(regex);\n parts.forEach(function(part, i) {\n if (i % 2 === 0) {\n frag.appendChild(document.createTextNode(part));\n } else {\n var span = document.createElement('span');\n span.className = 'userscript-highlight';\n span.textContent = part;\n frag.appendChild(span);\n }\n });\n node.parentNode.replaceChild(frag, node);\n }\n });\n } else if (node.nodeType === 1 && node.childNodes) { // element\n var skipTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEXTAREA', 'INPUT', 'SELECT'];\n if (!skipTags.includes(node.tagName)) {\n Array.from(node.childNodes).forEach(highlight);\n }\n }\n }\n \n highlight(document.body);\n \n // Re-highlight on dynamic content\n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1 || node.nodeType === 3) highlight(node);\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Highlight Search Terms");
}
} catch(__e) { console.warn('[Userscript:Highlight Search Terms]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
Perform field selections on lite messages by reading from the wire format#668

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

copybara-service
merged 1 commit into

from

Apr 25, 2025
 Merged


Perform field selections on lite messages by reading from the wire format
#668
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


1 change: 1 addition & 0 deletions
common/src/main/java/dev/cel/common/values/BUILD.bazel

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -179,6 +179,7 @@ java_library(
 "//protobuf:cel_lite_descriptor",
 "@maven//:com_google_errorprone_error_prone_annotations",
 "@maven//:com_google_guava_guava",
 "@maven//:com_google_protobuf_protobuf_java",
 "@maven//:org_jspecify_jspecify",
 "@maven_android//:com_google_protobuf_protobuf_javalite",
 ],
Expand Down

283 changes: 283 additions & 0 deletions
common/src/main/java/dev/cel/common/values/ProtoLiteCelValueConverter.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -16,12 +16,31 @@

import static com.google.common.base.Preconditions.checkNotNull;

import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Defaults;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.primitives.UnsignedLong;
import com.google.errorprone.annotations.Immutable;
import com.google.protobuf.ByteString;
import com.google.protobuf.CodedInputStream;
import com.google.protobuf.ExtensionRegistryLite;
import com.google.protobuf.MessageLite;
import com.google.protobuf.WireFormat;
import dev.cel.common.annotations.Internal;
import dev.cel.common.internal.CelLiteDescriptorPool;
import dev.cel.common.internal.WellKnownProto;
import dev.cel.protobuf.CelLiteDescriptor.FieldLiteDescriptor;
import dev.cel.protobuf.CelLiteDescriptor.FieldLiteDescriptor.CelFieldValueType;
import dev.cel.protobuf.CelLiteDescriptor.FieldLiteDescriptor.JavaType;
import dev.cel.protobuf.CelLiteDescriptor.MessageLiteDescriptor;
import java.io.IOException;
import java.util.AbstractMap;
import java.util.ArrayList;
import java.util.Collection;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;

/**
 * {@code ProtoLiteCelValueConverter} handles bidirectional conversion between native Java and
Expand All@@ -43,6 +62,270 @@ public static ProtoLiteCelValueConverter newInstance(
 return new ProtoLiteCelValueConverter(celLiteDescriptorPool);
 }

 private static Object readPrimitiveField(
 CodedInputStream inputStream, FieldLiteDescriptor fieldDescriptor) throws IOException {
 switch (fieldDescriptor.getProtoFieldType()) {
 case SINT32:
 return inputStream.readSInt32();
 case SINT64:
 return inputStream.readSInt64();
 case INT32:
 case ENUM:
 return inputStream.readInt32();
 case INT64:
 return inputStream.readInt64();
 case UINT32:
 return UnsignedLong.fromLongBits(inputStream.readUInt32());
 case UINT64:
 return UnsignedLong.fromLongBits(inputStream.readUInt64());
 case BOOL:
 return inputStream.readBool();
 case FLOAT:
 case FIXED32:
 case SFIXED32:
 return readFixed32BitField(inputStream, fieldDescriptor);
 case DOUBLE:
 case FIXED64:
 case SFIXED64:
 return readFixed64BitField(inputStream, fieldDescriptor);
 default:
 throw new IllegalStateException(
 "Unexpected field type: " + fieldDescriptor.getProtoFieldType());
 }
 }

 private static Object readFixed32BitField(
 CodedInputStream inputStream, FieldLiteDescriptor fieldDescriptor) throws IOException {
 switch (fieldDescriptor.getProtoFieldType()) {
 case FLOAT:
 return inputStream.readFloat();
 case FIXED32:
 case SFIXED32:
 return inputStream.readRawLittleEndian32();
 default:
 throw new IllegalStateException(
 "Unexpected field type: " + fieldDescriptor.getProtoFieldType());
 }
 }

 private static Object readFixed64BitField(
 CodedInputStream inputStream, FieldLiteDescriptor fieldDescriptor) throws IOException {
 switch (fieldDescriptor.getProtoFieldType()) {
 case DOUBLE:
 return inputStream.readDouble();
 case FIXED64:
 case SFIXED64:
 return inputStream.readRawLittleEndian64();
 default:
 throw new IllegalStateException(
 "Unexpected field type: " + fieldDescriptor.getProtoFieldType());
 }
 }

 private Object readLengthDelimitedField(
 CodedInputStream inputStream, FieldLiteDescriptor fieldDescriptor) throws IOException {
 FieldLiteDescriptor.Type fieldType = fieldDescriptor.getProtoFieldType();

 switch (fieldType) {
 case BYTES:
 return inputStream.readBytes();
 case MESSAGE:
 MessageLite.Builder builder =
 getDefaultMessageBuilder(fieldDescriptor.getFieldProtoTypeName());

 inputStream.readMessage(builder, ExtensionRegistryLite.getEmptyRegistry());
 return builder.build();
 case STRING:
 return inputStream.readStringRequireUtf8();
 default:
 throw new IllegalStateException("Unexpected field type: " + fieldType);
 }
 }

 private MessageLite.Builder getDefaultMessageBuilder(String protoTypeName) {
 return descriptorPool.getDescriptorOrThrow(protoTypeName).newMessageBuilder();
 }

 CelValue getDefaultCelValue(String protoTypeName, String fieldName) {
 MessageLiteDescriptor messageDescriptor = descriptorPool.getDescriptorOrThrow(protoTypeName);
 FieldLiteDescriptor fieldDescriptor = messageDescriptor.getByFieldNameOrThrow(fieldName);

 Object defaultValue = getDefaultValue(fieldDescriptor);
 if (defaultValue instanceof MessageLite) {
 return fromProtoMessageToCelValue(
 fieldDescriptor.getFieldProtoTypeName(), (MessageLite) defaultValue);
 } else {
 return fromJavaObjectToCelValue(getDefaultValue(fieldDescriptor));
 }
 }

 private Object getDefaultValue(FieldLiteDescriptor fieldDescriptor) {
 FieldLiteDescriptor.CelFieldValueType celFieldValueType =
 fieldDescriptor.getCelFieldValueType();
 switch (celFieldValueType) {
 case LIST:
 return ImmutableList.of();
 case MAP:
 return ImmutableMap.of();
 case SCALAR:
 return getScalarDefaultValue(fieldDescriptor);
 }
 throw new IllegalStateException("Unexpected cel field value type: " + celFieldValueType);
 }

 private Object getScalarDefaultValue(FieldLiteDescriptor fieldDescriptor) {
 JavaType type = fieldDescriptor.getJavaType();
 switch (type) {
 case INT:
 return fieldDescriptor.getProtoFieldType().equals(FieldLiteDescriptor.Type.UINT32)
 ? UnsignedLong.ZERO
 : Defaults.defaultValue(long.class);
 case LONG:
 return fieldDescriptor.getProtoFieldType().equals(FieldLiteDescriptor.Type.UINT64)
 ? UnsignedLong.ZERO
 : Defaults.defaultValue(long.class);
 case ENUM:
 return Defaults.defaultValue(long.class);
 case FLOAT:
 return Defaults.defaultValue(float.class);
 case DOUBLE:
 return Defaults.defaultValue(double.class);
 case BOOLEAN:
 return Defaults.defaultValue(boolean.class);
 case STRING:
 return "";
 case BYTE_STRING:
 return ByteString.EMPTY;
 case MESSAGE:
 if (WellKnownProto.isWrapperType(fieldDescriptor.getFieldProtoTypeName())) {
 return NullValue.NULL_VALUE;
 }

 return getDefaultMessageBuilder(fieldDescriptor.getFieldProtoTypeName()).build();
 }
 throw new IllegalStateException("Unexpected java type: " + type);
 }

 private ImmutableList<Object> readPackedRepeatedFields(
 CodedInputStream inputStream, FieldLiteDescriptor fieldDescriptor) throws IOException {
 int length = inputStream.readInt32();
 int oldLimit = inputStream.pushLimit(length);
 ImmutableList.Builder<Object> builder = ImmutableList.builder();
 while (inputStream.getBytesUntilLimit() > 0) {
 builder.add(readPrimitiveField(inputStream, fieldDescriptor));
 }
 inputStream.popLimit(oldLimit);
 return builder.build();
 }

 private Map.Entry<Object, Object> readSingleMapEntry(
 CodedInputStream inputStream, FieldLiteDescriptor fieldDescriptor) throws IOException {
 ImmutableMap<String, Object> singleMapEntry =
 readAllFields(inputStream.readByteArray(), fieldDescriptor.getFieldProtoTypeName());
 Object key = checkNotNull(singleMapEntry.get("key"));
 Object value = checkNotNull(singleMapEntry.get("value"));

 return new AbstractMap.SimpleEntry<>(key, value);
 }

 @VisibleForTesting
 ImmutableMap<String, Object> readAllFields(byte[] bytes, String protoTypeName)
 throws IOException {
 // TODO: Handle unknown fields by collecting them into a separate map.
 MessageLiteDescriptor messageDescriptor = descriptorPool.getDescriptorOrThrow(protoTypeName);
 CodedInputStream inputStream = CodedInputStream.newInstance(bytes);

 ImmutableMap.Builder<String, Object> fieldValues = ImmutableMap.builder();
 Map<Integer, List<Object>> repeatedFieldValues = new LinkedHashMap<>();
 Map<Integer, Map<Object, Object>> mapFieldValues = new LinkedHashMap<>();
 for (int tag = inputStream.readTag(); tag != 0; tag = inputStream.readTag()) {
 int tagWireType = WireFormat.getTagWireType(tag);
 int fieldNumber = WireFormat.getTagFieldNumber(tag);
 FieldLiteDescriptor fieldDescriptor = messageDescriptor.getByFieldNumberOrThrow(fieldNumber);

 Object payload;
 switch (tagWireType) {
 case WireFormat.WIRETYPE_VARINT:
 payload = readPrimitiveField(inputStream, fieldDescriptor);
 break;
 case WireFormat.WIRETYPE_FIXED32:
 payload = readFixed32BitField(inputStream, fieldDescriptor);
 break;
 case WireFormat.WIRETYPE_FIXED64:
 payload = readFixed64BitField(inputStream, fieldDescriptor);
 break;
 case WireFormat.WIRETYPE_LENGTH_DELIMITED:
 CelFieldValueType celFieldValueType = fieldDescriptor.getCelFieldValueType();
 switch (celFieldValueType) {
 case LIST:
 if (fieldDescriptor.getIsPacked()) {
 payload = readPackedRepeatedFields(inputStream, fieldDescriptor);
 } else {
 FieldLiteDescriptor.Type protoFieldType = fieldDescriptor.getProtoFieldType();
 boolean isLenDelimited =
 protoFieldType.equals(FieldLiteDescriptor.Type.MESSAGE)
 || protoFieldType.equals(FieldLiteDescriptor.Type.STRING)
 || protoFieldType.equals(FieldLiteDescriptor.Type.BYTES);
 if (!isLenDelimited) {
 throw new IllegalStateException(
 "Unexpected field type encountered for LEN-Delimited record: "
 + protoFieldType);
 }

 payload = readLengthDelimitedField(inputStream, fieldDescriptor);
 }
 break;
 case MAP:
 Map<Object, Object> fieldMap =
 mapFieldValues.computeIfAbsent(fieldNumber, (unused) -> new LinkedHashMap<>());
 Map.Entry<Object, Object> mapEntry = readSingleMapEntry(inputStream, fieldDescriptor);
 fieldMap.put(mapEntry.getKey(), mapEntry.getValue());
 payload = fieldMap;
 break;
 default:
 payload = readLengthDelimitedField(inputStream, fieldDescriptor);
 break;
 }
 break;
 case WireFormat.WIRETYPE_START_GROUP:
 case WireFormat.WIRETYPE_END_GROUP:
 // TODO: Support groups
 throw new UnsupportedOperationException("Groups are not supported");
 default:
 throw new IllegalArgumentException("Unexpected wire type: " + tagWireType);
 }

 if (fieldDescriptor.getCelFieldValueType().equals(CelFieldValueType.LIST)) {
 String fieldName = fieldDescriptor.getFieldName();
 List<Object> repeatedValues =
 repeatedFieldValues.computeIfAbsent(
 fieldNumber,
 (unused) -> {
 List<Object> newList = new ArrayList<>();
 fieldValues.put(fieldName, newList);
 return newList;
 });

 if (payload instanceof Collection) {
 repeatedValues.addAll((Collection<?>) payload);
 } else {
 repeatedValues.add(payload);
 }
 } else {
 fieldValues.put(fieldDescriptor.getFieldName(), payload);
 }
 }

 // Protobuf encoding follows a "last one wins" semantics. This means for duplicated fields,
 // we accept the last value encountered.
 return fieldValues.buildKeepingLast();
 }

 ImmutableMap<String, Object> readAllFields(MessageLite msg, String protoTypeName)
 throws IOException {
 return readAllFields(msg.toByteArray(), protoTypeName);
 }

 @Override
 public CelValue fromProtoMessageToCelValue(String protoTypeName, MessageLite msg) {
 checkNotNull(msg);
Expand Down

20 changes: 18 additions & 2 deletions
common/src/main/java/dev/cel/common/values/ProtoMessageLiteValue.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -15,11 +15,14 @@
package dev.cel.common.values;

import com.google.auto.value.AutoValue;
import com.google.auto.value.extension.memoized.Memoized;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableMap;
import com.google.errorprone.annotations.Immutable;
import com.google.protobuf.MessageLite;
import dev.cel.common.types.CelType;
import dev.cel.common.types.StructTypeReference;
import java.io.IOException;
import java.util.Optional;

/**
Expand All@@ -42,19 +45,32 @@ public abstract class ProtoMessageLiteValue extends StructValue<StringValue> {

 abstract ProtoLiteCelValueConverter protoLiteCelValueConverter();

 @Memoized
 ImmutableMap<String, Object> fieldValues() {
 try {
 return protoLiteCelValueConverter().readAllFields(value(), celType().name());
 } catch (IOException e) {
 throw new IllegalStateException("Unable to read message fields for " + celType().name(), e);
 }
 }

 @Override
 public boolean isZeroValue() {
 return value().getDefaultInstanceForType().equals(value());
 }

 @Override
 public CelValue select(StringValue field) {
 throw new UnsupportedOperationException("Not implemented yet");
 return find(field)
 .orElseGet(
 () -> protoLiteCelValueConverter().getDefaultCelValue(celType().name(), field.value()));
 }

 @Override
 public Optional<CelValue> find(StringValue field) {
 throw new UnsupportedOperationException("Not implemented yet");
 Object fieldValue = fieldValues().get(field.value());
 return Optional.ofNullable(fieldValue)
 .map(value -> protoLiteCelValueConverter().fromJavaObjectToCelValue(fieldValue));
 }

 public static ProtoMessageLiteValue create(
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Strip utm_, fbclid, gclid, etc. from all links on page\n(function() {\n var trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content',\n 'fbclid', 'gclid', 'dclid', 'msclkid', 'yclid',\n 'ref', 'ref_src', 'source', 'medium', 'campaign'];\n \n function cleanUrl(url) {\n try {\n var u = new URL(url, window.location.origin);\n var changed = false;\n trackingParams.forEach(function(p) {\n if (u.searchParams.has(p)) {\n u.searchParams.delete(p);\n changed = true;\n }\n });\n return changed ? u.toString() : url;\n } catch (e) {\n return url;\n }\n }\n \n function cleanLinks() {\n document.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n \n cleanLinks();\n \n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1) {\n if (node.tagName === 'A') cleanLinks();\n node.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Remove Tracking Parameters from Links");
}
} catch(__e) { console.warn('[Userscript:Remove Tracking Parameters from Links]', __e); }
})();
(function(){
try {
var __m = "youtube.com";
var __re = new RegExp('^' + "youtube\\.com" + '

Skip to content
Perform field selections on lite messages by reading from the wire format#668

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

copybara-service
merged 1 commit into

from

Apr 25, 2025
 Merged


Perform field selections on lite messages by reading from the wire format
#668
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


1 change: 1 addition & 0 deletions
common/src/main/java/dev/cel/common/values/BUILD.bazel

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -179,6 +179,7 @@ java_library(
 "//protobuf:cel_lite_descriptor",
 "@maven//:com_google_errorprone_error_prone_annotations",
 "@maven//:com_google_guava_guava",
 "@maven//:com_google_protobuf_protobuf_java",
 "@maven//:org_jspecify_jspecify",
 "@maven_android//:com_google_protobuf_protobuf_javalite",
 ],
Expand Down

283 changes: 283 additions & 0 deletions
common/src/main/java/dev/cel/common/values/ProtoLiteCelValueConverter.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -16,12 +16,31 @@

import static com.google.common.base.Preconditions.checkNotNull;

import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Defaults;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.primitives.UnsignedLong;
import com.google.errorprone.annotations.Immutable;
import com.google.protobuf.ByteString;
import com.google.protobuf.CodedInputStream;
import com.google.protobuf.ExtensionRegistryLite;
import com.google.protobuf.MessageLite;
import com.google.protobuf.WireFormat;
import dev.cel.common.annotations.Internal;
import dev.cel.common.internal.CelLiteDescriptorPool;
import dev.cel.common.internal.WellKnownProto;
import dev.cel.protobuf.CelLiteDescriptor.FieldLiteDescriptor;
import dev.cel.protobuf.CelLiteDescriptor.FieldLiteDescriptor.CelFieldValueType;
import dev.cel.protobuf.CelLiteDescriptor.FieldLiteDescriptor.JavaType;
import dev.cel.protobuf.CelLiteDescriptor.MessageLiteDescriptor;
import java.io.IOException;
import java.util.AbstractMap;
import java.util.ArrayList;
import java.util.Collection;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;

/**
 * {@code ProtoLiteCelValueConverter} handles bidirectional conversion between native Java and
Expand All@@ -43,6 +62,270 @@ public static ProtoLiteCelValueConverter newInstance(
 return new ProtoLiteCelValueConverter(celLiteDescriptorPool);
 }

 private static Object readPrimitiveField(
 CodedInputStream inputStream, FieldLiteDescriptor fieldDescriptor) throws IOException {
 switch (fieldDescriptor.getProtoFieldType()) {
 case SINT32:
 return inputStream.readSInt32();
 case SINT64:
 return inputStream.readSInt64();
 case INT32:
 case ENUM:
 return inputStream.readInt32();
 case INT64:
 return inputStream.readInt64();
 case UINT32:
 return UnsignedLong.fromLongBits(inputStream.readUInt32());
 case UINT64:
 return UnsignedLong.fromLongBits(inputStream.readUInt64());
 case BOOL:
 return inputStream.readBool();
 case FLOAT:
 case FIXED32:
 case SFIXED32:
 return readFixed32BitField(inputStream, fieldDescriptor);
 case DOUBLE:
 case FIXED64:
 case SFIXED64:
 return readFixed64BitField(inputStream, fieldDescriptor);
 default:
 throw new IllegalStateException(
 "Unexpected field type: " + fieldDescriptor.getProtoFieldType());
 }
 }

 private static Object readFixed32BitField(
 CodedInputStream inputStream, FieldLiteDescriptor fieldDescriptor) throws IOException {
 switch (fieldDescriptor.getProtoFieldType()) {
 case FLOAT:
 return inputStream.readFloat();
 case FIXED32:
 case SFIXED32:
 return inputStream.readRawLittleEndian32();
 default:
 throw new IllegalStateException(
 "Unexpected field type: " + fieldDescriptor.getProtoFieldType());
 }
 }

 private static Object readFixed64BitField(
 CodedInputStream inputStream, FieldLiteDescriptor fieldDescriptor) throws IOException {
 switch (fieldDescriptor.getProtoFieldType()) {
 case DOUBLE:
 return inputStream.readDouble();
 case FIXED64:
 case SFIXED64:
 return inputStream.readRawLittleEndian64();
 default:
 throw new IllegalStateException(
 "Unexpected field type: " + fieldDescriptor.getProtoFieldType());
 }
 }

 private Object readLengthDelimitedField(
 CodedInputStream inputStream, FieldLiteDescriptor fieldDescriptor) throws IOException {
 FieldLiteDescriptor.Type fieldType = fieldDescriptor.getProtoFieldType();

 switch (fieldType) {
 case BYTES:
 return inputStream.readBytes();
 case MESSAGE:
 MessageLite.Builder builder =
 getDefaultMessageBuilder(fieldDescriptor.getFieldProtoTypeName());

 inputStream.readMessage(builder, ExtensionRegistryLite.getEmptyRegistry());
 return builder.build();
 case STRING:
 return inputStream.readStringRequireUtf8();
 default:
 throw new IllegalStateException("Unexpected field type: " + fieldType);
 }
 }

 private MessageLite.Builder getDefaultMessageBuilder(String protoTypeName) {
 return descriptorPool.getDescriptorOrThrow(protoTypeName).newMessageBuilder();
 }

 CelValue getDefaultCelValue(String protoTypeName, String fieldName) {
 MessageLiteDescriptor messageDescriptor = descriptorPool.getDescriptorOrThrow(protoTypeName);
 FieldLiteDescriptor fieldDescriptor = messageDescriptor.getByFieldNameOrThrow(fieldName);

 Object defaultValue = getDefaultValue(fieldDescriptor);
 if (defaultValue instanceof MessageLite) {
 return fromProtoMessageToCelValue(
 fieldDescriptor.getFieldProtoTypeName(), (MessageLite) defaultValue);
 } else {
 return fromJavaObjectToCelValue(getDefaultValue(fieldDescriptor));
 }
 }

 private Object getDefaultValue(FieldLiteDescriptor fieldDescriptor) {
 FieldLiteDescriptor.CelFieldValueType celFieldValueType =
 fieldDescriptor.getCelFieldValueType();
 switch (celFieldValueType) {
 case LIST:
 return ImmutableList.of();
 case MAP:
 return ImmutableMap.of();
 case SCALAR:
 return getScalarDefaultValue(fieldDescriptor);
 }
 throw new IllegalStateException("Unexpected cel field value type: " + celFieldValueType);
 }

 private Object getScalarDefaultValue(FieldLiteDescriptor fieldDescriptor) {
 JavaType type = fieldDescriptor.getJavaType();
 switch (type) {
 case INT:
 return fieldDescriptor.getProtoFieldType().equals(FieldLiteDescriptor.Type.UINT32)
 ? UnsignedLong.ZERO
 : Defaults.defaultValue(long.class);
 case LONG:
 return fieldDescriptor.getProtoFieldType().equals(FieldLiteDescriptor.Type.UINT64)
 ? UnsignedLong.ZERO
 : Defaults.defaultValue(long.class);
 case ENUM:
 return Defaults.defaultValue(long.class);
 case FLOAT:
 return Defaults.defaultValue(float.class);
 case DOUBLE:
 return Defaults.defaultValue(double.class);
 case BOOLEAN:
 return Defaults.defaultValue(boolean.class);
 case STRING:
 return "";
 case BYTE_STRING:
 return ByteString.EMPTY;
 case MESSAGE:
 if (WellKnownProto.isWrapperType(fieldDescriptor.getFieldProtoTypeName())) {
 return NullValue.NULL_VALUE;
 }

 return getDefaultMessageBuilder(fieldDescriptor.getFieldProtoTypeName()).build();
 }
 throw new IllegalStateException("Unexpected java type: " + type);
 }

 private ImmutableList<Object> readPackedRepeatedFields(
 CodedInputStream inputStream, FieldLiteDescriptor fieldDescriptor) throws IOException {
 int length = inputStream.readInt32();
 int oldLimit = inputStream.pushLimit(length);
 ImmutableList.Builder<Object> builder = ImmutableList.builder();
 while (inputStream.getBytesUntilLimit() > 0) {
 builder.add(readPrimitiveField(inputStream, fieldDescriptor));
 }
 inputStream.popLimit(oldLimit);
 return builder.build();
 }

 private Map.Entry<Object, Object> readSingleMapEntry(
 CodedInputStream inputStream, FieldLiteDescriptor fieldDescriptor) throws IOException {
 ImmutableMap<String, Object> singleMapEntry =
 readAllFields(inputStream.readByteArray(), fieldDescriptor.getFieldProtoTypeName());
 Object key = checkNotNull(singleMapEntry.get("key"));
 Object value = checkNotNull(singleMapEntry.get("value"));

 return new AbstractMap.SimpleEntry<>(key, value);
 }

 @VisibleForTesting
 ImmutableMap<String, Object> readAllFields(byte[] bytes, String protoTypeName)
 throws IOException {
 // TODO: Handle unknown fields by collecting them into a separate map.
 MessageLiteDescriptor messageDescriptor = descriptorPool.getDescriptorOrThrow(protoTypeName);
 CodedInputStream inputStream = CodedInputStream.newInstance(bytes);

 ImmutableMap.Builder<String, Object> fieldValues = ImmutableMap.builder();
 Map<Integer, List<Object>> repeatedFieldValues = new LinkedHashMap<>();
 Map<Integer, Map<Object, Object>> mapFieldValues = new LinkedHashMap<>();
 for (int tag = inputStream.readTag(); tag != 0; tag = inputStream.readTag()) {
 int tagWireType = WireFormat.getTagWireType(tag);
 int fieldNumber = WireFormat.getTagFieldNumber(tag);
 FieldLiteDescriptor fieldDescriptor = messageDescriptor.getByFieldNumberOrThrow(fieldNumber);

 Object payload;
 switch (tagWireType) {
 case WireFormat.WIRETYPE_VARINT:
 payload = readPrimitiveField(inputStream, fieldDescriptor);
 break;
 case WireFormat.WIRETYPE_FIXED32:
 payload = readFixed32BitField(inputStream, fieldDescriptor);
 break;
 case WireFormat.WIRETYPE_FIXED64:
 payload = readFixed64BitField(inputStream, fieldDescriptor);
 break;
 case WireFormat.WIRETYPE_LENGTH_DELIMITED:
 CelFieldValueType celFieldValueType = fieldDescriptor.getCelFieldValueType();
 switch (celFieldValueType) {
 case LIST:
 if (fieldDescriptor.getIsPacked()) {
 payload = readPackedRepeatedFields(inputStream, fieldDescriptor);
 } else {
 FieldLiteDescriptor.Type protoFieldType = fieldDescriptor.getProtoFieldType();
 boolean isLenDelimited =
 protoFieldType.equals(FieldLiteDescriptor.Type.MESSAGE)
 || protoFieldType.equals(FieldLiteDescriptor.Type.STRING)
 || protoFieldType.equals(FieldLiteDescriptor.Type.BYTES);
 if (!isLenDelimited) {
 throw new IllegalStateException(
 "Unexpected field type encountered for LEN-Delimited record: "
 + protoFieldType);
 }

 payload = readLengthDelimitedField(inputStream, fieldDescriptor);
 }
 break;
 case MAP:
 Map<Object, Object> fieldMap =
 mapFieldValues.computeIfAbsent(fieldNumber, (unused) -> new LinkedHashMap<>());
 Map.Entry<Object, Object> mapEntry = readSingleMapEntry(inputStream, fieldDescriptor);
 fieldMap.put(mapEntry.getKey(), mapEntry.getValue());
 payload = fieldMap;
 break;
 default:
 payload = readLengthDelimitedField(inputStream, fieldDescriptor);
 break;
 }
 break;
 case WireFormat.WIRETYPE_START_GROUP:
 case WireFormat.WIRETYPE_END_GROUP:
 // TODO: Support groups
 throw new UnsupportedOperationException("Groups are not supported");
 default:
 throw new IllegalArgumentException("Unexpected wire type: " + tagWireType);
 }

 if (fieldDescriptor.getCelFieldValueType().equals(CelFieldValueType.LIST)) {
 String fieldName = fieldDescriptor.getFieldName();
 List<Object> repeatedValues =
 repeatedFieldValues.computeIfAbsent(
 fieldNumber,
 (unused) -> {
 List<Object> newList = new ArrayList<>();
 fieldValues.put(fieldName, newList);
 return newList;
 });

 if (payload instanceof Collection) {
 repeatedValues.addAll((Collection<?>) payload);
 } else {
 repeatedValues.add(payload);
 }
 } else {
 fieldValues.put(fieldDescriptor.getFieldName(), payload);
 }
 }

 // Protobuf encoding follows a "last one wins" semantics. This means for duplicated fields,
 // we accept the last value encountered.
 return fieldValues.buildKeepingLast();
 }

 ImmutableMap<String, Object> readAllFields(MessageLite msg, String protoTypeName)
 throws IOException {
 return readAllFields(msg.toByteArray(), protoTypeName);
 }

 @Override
 public CelValue fromProtoMessageToCelValue(String protoTypeName, MessageLite msg) {
 checkNotNull(msg);
Expand Down

20 changes: 18 additions & 2 deletions
common/src/main/java/dev/cel/common/values/ProtoMessageLiteValue.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -15,11 +15,14 @@
package dev.cel.common.values;

import com.google.auto.value.AutoValue;
import com.google.auto.value.extension.memoized.Memoized;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableMap;
import com.google.errorprone.annotations.Immutable;
import com.google.protobuf.MessageLite;
import dev.cel.common.types.CelType;
import dev.cel.common.types.StructTypeReference;
import java.io.IOException;
import java.util.Optional;

/**
Expand All@@ -42,19 +45,32 @@ public abstract class ProtoMessageLiteValue extends StructValue<StringValue> {

 abstract ProtoLiteCelValueConverter protoLiteCelValueConverter();

 @Memoized
 ImmutableMap<String, Object> fieldValues() {
 try {
 return protoLiteCelValueConverter().readAllFields(value(), celType().name());
 } catch (IOException e) {
 throw new IllegalStateException("Unable to read message fields for " + celType().name(), e);
 }
 }

 @Override
 public boolean isZeroValue() {
 return value().getDefaultInstanceForType().equals(value());
 }

 @Override
 public CelValue select(StringValue field) {
 throw new UnsupportedOperationException("Not implemented yet");
 return find(field)
 .orElseGet(
 () -> protoLiteCelValueConverter().getDefaultCelValue(celType().name(), field.value()));
 }

 @Override
 public Optional<CelValue> find(StringValue field) {
 throw new UnsupportedOperationException("Not implemented yet");
 Object fieldValue = fieldValues().get(field.value());
 return Optional.ofNullable(fieldValue)
 .map(value -> protoLiteCelValueConverter().fromJavaObjectToCelValue(fieldValue));
 }

 public static ProtoMessageLiteValue create(
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Auto-enable theater mode on YouTube\n(function() {\n function tryTheater() {\n var btn = document.querySelector('button[aria-label=\"Theater mode\"], ytd-player #player button[title=\"Theater mode\"]');\n if (btn && !btn.classList.contains('activated')) {\n btn.click();\n }\n }\n \n // Try immediately\n tryTheater();\n \n // Try after navigation (SPA)\n var lastUrl = location.href;\n setInterval(function() {\n if (location.href !== lastUrl) {\n lastUrl = location.href;\n setTimeout(tryTheater, 500);\n }\n }, 1000);\n \n // Also try on player load\n var observer = new MutationObserver(tryTheater);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "YouTube Theater Mode Default");
}
} catch(__e) { console.warn('[Userscript:YouTube Theater Mode Default]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
Perform field selections on lite messages by reading from the wire format#668

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

copybara-service
merged 1 commit into

from

Apr 25, 2025
 Merged


Perform field selections on lite messages by reading from the wire format
#668
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


1 change: 1 addition & 0 deletions
common/src/main/java/dev/cel/common/values/BUILD.bazel

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -179,6 +179,7 @@ java_library(
 "//protobuf:cel_lite_descriptor",
 "@maven//:com_google_errorprone_error_prone_annotations",
 "@maven//:com_google_guava_guava",
 "@maven//:com_google_protobuf_protobuf_java",
 "@maven//:org_jspecify_jspecify",
 "@maven_android//:com_google_protobuf_protobuf_javalite",
 ],
Expand Down

283 changes: 283 additions & 0 deletions
common/src/main/java/dev/cel/common/values/ProtoLiteCelValueConverter.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -16,12 +16,31 @@

import static com.google.common.base.Preconditions.checkNotNull;

import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Defaults;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.primitives.UnsignedLong;
import com.google.errorprone.annotations.Immutable;
import com.google.protobuf.ByteString;
import com.google.protobuf.CodedInputStream;
import com.google.protobuf.ExtensionRegistryLite;
import com.google.protobuf.MessageLite;
import com.google.protobuf.WireFormat;
import dev.cel.common.annotations.Internal;
import dev.cel.common.internal.CelLiteDescriptorPool;
import dev.cel.common.internal.WellKnownProto;
import dev.cel.protobuf.CelLiteDescriptor.FieldLiteDescriptor;
import dev.cel.protobuf.CelLiteDescriptor.FieldLiteDescriptor.CelFieldValueType;
import dev.cel.protobuf.CelLiteDescriptor.FieldLiteDescriptor.JavaType;
import dev.cel.protobuf.CelLiteDescriptor.MessageLiteDescriptor;
import java.io.IOException;
import java.util.AbstractMap;
import java.util.ArrayList;
import java.util.Collection;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;

/**
 * {@code ProtoLiteCelValueConverter} handles bidirectional conversion between native Java and
Expand All@@ -43,6 +62,270 @@ public static ProtoLiteCelValueConverter newInstance(
 return new ProtoLiteCelValueConverter(celLiteDescriptorPool);
 }

 private static Object readPrimitiveField(
 CodedInputStream inputStream, FieldLiteDescriptor fieldDescriptor) throws IOException {
 switch (fieldDescriptor.getProtoFieldType()) {
 case SINT32:
 return inputStream.readSInt32();
 case SINT64:
 return inputStream.readSInt64();
 case INT32:
 case ENUM:
 return inputStream.readInt32();
 case INT64:
 return inputStream.readInt64();
 case UINT32:
 return UnsignedLong.fromLongBits(inputStream.readUInt32());
 case UINT64:
 return UnsignedLong.fromLongBits(inputStream.readUInt64());
 case BOOL:
 return inputStream.readBool();
 case FLOAT:
 case FIXED32:
 case SFIXED32:
 return readFixed32BitField(inputStream, fieldDescriptor);
 case DOUBLE:
 case FIXED64:
 case SFIXED64:
 return readFixed64BitField(inputStream, fieldDescriptor);
 default:
 throw new IllegalStateException(
 "Unexpected field type: " + fieldDescriptor.getProtoFieldType());
 }
 }

 private static Object readFixed32BitField(
 CodedInputStream inputStream, FieldLiteDescriptor fieldDescriptor) throws IOException {
 switch (fieldDescriptor.getProtoFieldType()) {
 case FLOAT:
 return inputStream.readFloat();
 case FIXED32:
 case SFIXED32:
 return inputStream.readRawLittleEndian32();
 default:
 throw new IllegalStateException(
 "Unexpected field type: " + fieldDescriptor.getProtoFieldType());
 }
 }

 private static Object readFixed64BitField(
 CodedInputStream inputStream, FieldLiteDescriptor fieldDescriptor) throws IOException {
 switch (fieldDescriptor.getProtoFieldType()) {
 case DOUBLE:
 return inputStream.readDouble();
 case FIXED64:
 case SFIXED64:
 return inputStream.readRawLittleEndian64();
 default:
 throw new IllegalStateException(
 "Unexpected field type: " + fieldDescriptor.getProtoFieldType());
 }
 }

 private Object readLengthDelimitedField(
 CodedInputStream inputStream, FieldLiteDescriptor fieldDescriptor) throws IOException {
 FieldLiteDescriptor.Type fieldType = fieldDescriptor.getProtoFieldType();

 switch (fieldType) {
 case BYTES:
 return inputStream.readBytes();
 case MESSAGE:
 MessageLite.Builder builder =
 getDefaultMessageBuilder(fieldDescriptor.getFieldProtoTypeName());

 inputStream.readMessage(builder, ExtensionRegistryLite.getEmptyRegistry());
 return builder.build();
 case STRING:
 return inputStream.readStringRequireUtf8();
 default:
 throw new IllegalStateException("Unexpected field type: " + fieldType);
 }
 }

 private MessageLite.Builder getDefaultMessageBuilder(String protoTypeName) {
 return descriptorPool.getDescriptorOrThrow(protoTypeName).newMessageBuilder();
 }

 CelValue getDefaultCelValue(String protoTypeName, String fieldName) {
 MessageLiteDescriptor messageDescriptor = descriptorPool.getDescriptorOrThrow(protoTypeName);
 FieldLiteDescriptor fieldDescriptor = messageDescriptor.getByFieldNameOrThrow(fieldName);

 Object defaultValue = getDefaultValue(fieldDescriptor);
 if (defaultValue instanceof MessageLite) {
 return fromProtoMessageToCelValue(
 fieldDescriptor.getFieldProtoTypeName(), (MessageLite) defaultValue);
 } else {
 return fromJavaObjectToCelValue(getDefaultValue(fieldDescriptor));
 }
 }

 private Object getDefaultValue(FieldLiteDescriptor fieldDescriptor) {
 FieldLiteDescriptor.CelFieldValueType celFieldValueType =
 fieldDescriptor.getCelFieldValueType();
 switch (celFieldValueType) {
 case LIST:
 return ImmutableList.of();
 case MAP:
 return ImmutableMap.of();
 case SCALAR:
 return getScalarDefaultValue(fieldDescriptor);
 }
 throw new IllegalStateException("Unexpected cel field value type: " + celFieldValueType);
 }

 private Object getScalarDefaultValue(FieldLiteDescriptor fieldDescriptor) {
 JavaType type = fieldDescriptor.getJavaType();
 switch (type) {
 case INT:
 return fieldDescriptor.getProtoFieldType().equals(FieldLiteDescriptor.Type.UINT32)
 ? UnsignedLong.ZERO
 : Defaults.defaultValue(long.class);
 case LONG:
 return fieldDescriptor.getProtoFieldType().equals(FieldLiteDescriptor.Type.UINT64)
 ? UnsignedLong.ZERO
 : Defaults.defaultValue(long.class);
 case ENUM:
 return Defaults.defaultValue(long.class);
 case FLOAT:
 return Defaults.defaultValue(float.class);
 case DOUBLE:
 return Defaults.defaultValue(double.class);
 case BOOLEAN:
 return Defaults.defaultValue(boolean.class);
 case STRING:
 return "";
 case BYTE_STRING:
 return ByteString.EMPTY;
 case MESSAGE:
 if (WellKnownProto.isWrapperType(fieldDescriptor.getFieldProtoTypeName())) {
 return NullValue.NULL_VALUE;
 }

 return getDefaultMessageBuilder(fieldDescriptor.getFieldProtoTypeName()).build();
 }
 throw new IllegalStateException("Unexpected java type: " + type);
 }

 private ImmutableList<Object> readPackedRepeatedFields(
 CodedInputStream inputStream, FieldLiteDescriptor fieldDescriptor) throws IOException {
 int length = inputStream.readInt32();
 int oldLimit = inputStream.pushLimit(length);
 ImmutableList.Builder<Object> builder = ImmutableList.builder();
 while (inputStream.getBytesUntilLimit() > 0) {
 builder.add(readPrimitiveField(inputStream, fieldDescriptor));
 }
 inputStream.popLimit(oldLimit);
 return builder.build();
 }

 private Map.Entry<Object, Object> readSingleMapEntry(
 CodedInputStream inputStream, FieldLiteDescriptor fieldDescriptor) throws IOException {
 ImmutableMap<String, Object> singleMapEntry =
 readAllFields(inputStream.readByteArray(), fieldDescriptor.getFieldProtoTypeName());
 Object key = checkNotNull(singleMapEntry.get("key"));
 Object value = checkNotNull(singleMapEntry.get("value"));

 return new AbstractMap.SimpleEntry<>(key, value);
 }

 @VisibleForTesting
 ImmutableMap<String, Object> readAllFields(byte[] bytes, String protoTypeName)
 throws IOException {
 // TODO: Handle unknown fields by collecting them into a separate map.
 MessageLiteDescriptor messageDescriptor = descriptorPool.getDescriptorOrThrow(protoTypeName);
 CodedInputStream inputStream = CodedInputStream.newInstance(bytes);

 ImmutableMap.Builder<String, Object> fieldValues = ImmutableMap.builder();
 Map<Integer, List<Object>> repeatedFieldValues = new LinkedHashMap<>();
 Map<Integer, Map<Object, Object>> mapFieldValues = new LinkedHashMap<>();
 for (int tag = inputStream.readTag(); tag != 0; tag = inputStream.readTag()) {
 int tagWireType = WireFormat.getTagWireType(tag);
 int fieldNumber = WireFormat.getTagFieldNumber(tag);
 FieldLiteDescriptor fieldDescriptor = messageDescriptor.getByFieldNumberOrThrow(fieldNumber);

 Object payload;
 switch (tagWireType) {
 case WireFormat.WIRETYPE_VARINT:
 payload = readPrimitiveField(inputStream, fieldDescriptor);
 break;
 case WireFormat.WIRETYPE_FIXED32:
 payload = readFixed32BitField(inputStream, fieldDescriptor);
 break;
 case WireFormat.WIRETYPE_FIXED64:
 payload = readFixed64BitField(inputStream, fieldDescriptor);
 break;
 case WireFormat.WIRETYPE_LENGTH_DELIMITED:
 CelFieldValueType celFieldValueType = fieldDescriptor.getCelFieldValueType();
 switch (celFieldValueType) {
 case LIST:
 if (fieldDescriptor.getIsPacked()) {
 payload = readPackedRepeatedFields(inputStream, fieldDescriptor);
 } else {
 FieldLiteDescriptor.Type protoFieldType = fieldDescriptor.getProtoFieldType();
 boolean isLenDelimited =
 protoFieldType.equals(FieldLiteDescriptor.Type.MESSAGE)
 || protoFieldType.equals(FieldLiteDescriptor.Type.STRING)
 || protoFieldType.equals(FieldLiteDescriptor.Type.BYTES);
 if (!isLenDelimited) {
 throw new IllegalStateException(
 "Unexpected field type encountered for LEN-Delimited record: "
 + protoFieldType);
 }

 payload = readLengthDelimitedField(inputStream, fieldDescriptor);
 }
 break;
 case MAP:
 Map<Object, Object> fieldMap =
 mapFieldValues.computeIfAbsent(fieldNumber, (unused) -> new LinkedHashMap<>());
 Map.Entry<Object, Object> mapEntry = readSingleMapEntry(inputStream, fieldDescriptor);
 fieldMap.put(mapEntry.getKey(), mapEntry.getValue());
 payload = fieldMap;
 break;
 default:
 payload = readLengthDelimitedField(inputStream, fieldDescriptor);
 break;
 }
 break;
 case WireFormat.WIRETYPE_START_GROUP:
 case WireFormat.WIRETYPE_END_GROUP:
 // TODO: Support groups
 throw new UnsupportedOperationException("Groups are not supported");
 default:
 throw new IllegalArgumentException("Unexpected wire type: " + tagWireType);
 }

 if (fieldDescriptor.getCelFieldValueType().equals(CelFieldValueType.LIST)) {
 String fieldName = fieldDescriptor.getFieldName();
 List<Object> repeatedValues =
 repeatedFieldValues.computeIfAbsent(
 fieldNumber,
 (unused) -> {
 List<Object> newList = new ArrayList<>();
 fieldValues.put(fieldName, newList);
 return newList;
 });

 if (payload instanceof Collection) {
 repeatedValues.addAll((Collection<?>) payload);
 } else {
 repeatedValues.add(payload);
 }
 } else {
 fieldValues.put(fieldDescriptor.getFieldName(), payload);
 }
 }

 // Protobuf encoding follows a "last one wins" semantics. This means for duplicated fields,
 // we accept the last value encountered.
 return fieldValues.buildKeepingLast();
 }

 ImmutableMap<String, Object> readAllFields(MessageLite msg, String protoTypeName)
 throws IOException {
 return readAllFields(msg.toByteArray(), protoTypeName);
 }

 @Override
 public CelValue fromProtoMessageToCelValue(String protoTypeName, MessageLite msg) {
 checkNotNull(msg);
Expand Down

20 changes: 18 additions & 2 deletions
common/src/main/java/dev/cel/common/values/ProtoMessageLiteValue.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -15,11 +15,14 @@
package dev.cel.common.values;

import com.google.auto.value.AutoValue;
import com.google.auto.value.extension.memoized.Memoized;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableMap;
import com.google.errorprone.annotations.Immutable;
import com.google.protobuf.MessageLite;
import dev.cel.common.types.CelType;
import dev.cel.common.types.StructTypeReference;
import java.io.IOException;
import java.util.Optional;

/**
Expand All@@ -42,19 +45,32 @@ public abstract class ProtoMessageLiteValue extends StructValue<StringValue> {

 abstract ProtoLiteCelValueConverter protoLiteCelValueConverter();

 @Memoized
 ImmutableMap<String, Object> fieldValues() {
 try {
 return protoLiteCelValueConverter().readAllFields(value(), celType().name());
 } catch (IOException e) {
 throw new IllegalStateException("Unable to read message fields for " + celType().name(), e);
 }
 }

 @Override
 public boolean isZeroValue() {
 return value().getDefaultInstanceForType().equals(value());
 }

 @Override
 public CelValue select(StringValue field) {
 throw new UnsupportedOperationException("Not implemented yet");
 return find(field)
 .orElseGet(
 () -> protoLiteCelValueConverter().getDefaultCelValue(celType().name(), field.value()));
 }

 @Override
 public Optional<CelValue> find(StringValue field) {
 throw new UnsupportedOperationException("Not implemented yet");
 Object fieldValue = fieldValues().get(field.value());
 return Optional.ofNullable(fieldValue)
 .map(value -> protoLiteCelValueConverter().fromJavaObjectToCelValue(fieldValue));
 }

 public static ProtoMessageLiteValue create(
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Remove or un-stick sticky/fixed headers that block content\n(function() {\n function unstick() {\n document.querySelectorAll('header, nav, [role=\"banner\"], .header, .navbar, .sticky, .fixed-top, [style*=\"position: fixed\"], [style*=\"position:sticky\"]').forEach(function(el) {\n if (el.style.position === 'fixed' || el.style.position === 'sticky' || \n getComputedStyle(el).position === 'fixed' || getComputedStyle(el).position === 'sticky') {\n el.style.position = 'static';\n el.style.top = 'auto';\n el.style.zIndex = 'auto';\n }\n });\n }\n \n unstick();\n \n var observer = new MutationObserver(unstick);\n observer.observe(document.body, { childList: true, subtree: true, attributes: true, attributeFilter: ['style', 'class'] });\n})();", "Kill Sticky Headers");
}
} catch(__e) { console.warn('[Userscript:Kill Sticky Headers]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
Perform field selections on lite messages by reading from the wire format#668

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

copybara-service
merged 1 commit into

from

Apr 25, 2025
 Merged


Perform field selections on lite messages by reading from the wire format
#668
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


1 change: 1 addition & 0 deletions
common/src/main/java/dev/cel/common/values/BUILD.bazel

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -179,6 +179,7 @@ java_library(
 "//protobuf:cel_lite_descriptor",
 "@maven//:com_google_errorprone_error_prone_annotations",
 "@maven//:com_google_guava_guava",
 "@maven//:com_google_protobuf_protobuf_java",
 "@maven//:org_jspecify_jspecify",
 "@maven_android//:com_google_protobuf_protobuf_javalite",
 ],
Expand Down

283 changes: 283 additions & 0 deletions
common/src/main/java/dev/cel/common/values/ProtoLiteCelValueConverter.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -16,12 +16,31 @@

import static com.google.common.base.Preconditions.checkNotNull;

import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Defaults;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.primitives.UnsignedLong;
import com.google.errorprone.annotations.Immutable;
import com.google.protobuf.ByteString;
import com.google.protobuf.CodedInputStream;
import com.google.protobuf.ExtensionRegistryLite;
import com.google.protobuf.MessageLite;
import com.google.protobuf.WireFormat;
import dev.cel.common.annotations.Internal;
import dev.cel.common.internal.CelLiteDescriptorPool;
import dev.cel.common.internal.WellKnownProto;
import dev.cel.protobuf.CelLiteDescriptor.FieldLiteDescriptor;
import dev.cel.protobuf.CelLiteDescriptor.FieldLiteDescriptor.CelFieldValueType;
import dev.cel.protobuf.CelLiteDescriptor.FieldLiteDescriptor.JavaType;
import dev.cel.protobuf.CelLiteDescriptor.MessageLiteDescriptor;
import java.io.IOException;
import java.util.AbstractMap;
import java.util.ArrayList;
import java.util.Collection;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;

/**
 * {@code ProtoLiteCelValueConverter} handles bidirectional conversion between native Java and
Expand All@@ -43,6 +62,270 @@ public static ProtoLiteCelValueConverter newInstance(
 return new ProtoLiteCelValueConverter(celLiteDescriptorPool);
 }

 private static Object readPrimitiveField(
 CodedInputStream inputStream, FieldLiteDescriptor fieldDescriptor) throws IOException {
 switch (fieldDescriptor.getProtoFieldType()) {
 case SINT32:
 return inputStream.readSInt32();
 case SINT64:
 return inputStream.readSInt64();
 case INT32:
 case ENUM:
 return inputStream.readInt32();
 case INT64:
 return inputStream.readInt64();
 case UINT32:
 return UnsignedLong.fromLongBits(inputStream.readUInt32());
 case UINT64:
 return UnsignedLong.fromLongBits(inputStream.readUInt64());
 case BOOL:
 return inputStream.readBool();
 case FLOAT:
 case FIXED32:
 case SFIXED32:
 return readFixed32BitField(inputStream, fieldDescriptor);
 case DOUBLE:
 case FIXED64:
 case SFIXED64:
 return readFixed64BitField(inputStream, fieldDescriptor);
 default:
 throw new IllegalStateException(
 "Unexpected field type: " + fieldDescriptor.getProtoFieldType());
 }
 }

 private static Object readFixed32BitField(
 CodedInputStream inputStream, FieldLiteDescriptor fieldDescriptor) throws IOException {
 switch (fieldDescriptor.getProtoFieldType()) {
 case FLOAT:
 return inputStream.readFloat();
 case FIXED32:
 case SFIXED32:
 return inputStream.readRawLittleEndian32();
 default:
 throw new IllegalStateException(
 "Unexpected field type: " + fieldDescriptor.getProtoFieldType());
 }
 }

 private static Object readFixed64BitField(
 CodedInputStream inputStream, FieldLiteDescriptor fieldDescriptor) throws IOException {
 switch (fieldDescriptor.getProtoFieldType()) {
 case DOUBLE:
 return inputStream.readDouble();
 case FIXED64:
 case SFIXED64:
 return inputStream.readRawLittleEndian64();
 default:
 throw new IllegalStateException(
 "Unexpected field type: " + fieldDescriptor.getProtoFieldType());
 }
 }

 private Object readLengthDelimitedField(
 CodedInputStream inputStream, FieldLiteDescriptor fieldDescriptor) throws IOException {
 FieldLiteDescriptor.Type fieldType = fieldDescriptor.getProtoFieldType();

 switch (fieldType) {
 case BYTES:
 return inputStream.readBytes();
 case MESSAGE:
 MessageLite.Builder builder =
 getDefaultMessageBuilder(fieldDescriptor.getFieldProtoTypeName());

 inputStream.readMessage(builder, ExtensionRegistryLite.getEmptyRegistry());
 return builder.build();
 case STRING:
 return inputStream.readStringRequireUtf8();
 default:
 throw new IllegalStateException("Unexpected field type: " + fieldType);
 }
 }

 private MessageLite.Builder getDefaultMessageBuilder(String protoTypeName) {
 return descriptorPool.getDescriptorOrThrow(protoTypeName).newMessageBuilder();
 }

 CelValue getDefaultCelValue(String protoTypeName, String fieldName) {
 MessageLiteDescriptor messageDescriptor = descriptorPool.getDescriptorOrThrow(protoTypeName);
 FieldLiteDescriptor fieldDescriptor = messageDescriptor.getByFieldNameOrThrow(fieldName);

 Object defaultValue = getDefaultValue(fieldDescriptor);
 if (defaultValue instanceof MessageLite) {
 return fromProtoMessageToCelValue(
 fieldDescriptor.getFieldProtoTypeName(), (MessageLite) defaultValue);
 } else {
 return fromJavaObjectToCelValue(getDefaultValue(fieldDescriptor));
 }
 }

 private Object getDefaultValue(FieldLiteDescriptor fieldDescriptor) {
 FieldLiteDescriptor.CelFieldValueType celFieldValueType =
 fieldDescriptor.getCelFieldValueType();
 switch (celFieldValueType) {
 case LIST:
 return ImmutableList.of();
 case MAP:
 return ImmutableMap.of();
 case SCALAR:
 return getScalarDefaultValue(fieldDescriptor);
 }
 throw new IllegalStateException("Unexpected cel field value type: " + celFieldValueType);
 }

 private Object getScalarDefaultValue(FieldLiteDescriptor fieldDescriptor) {
 JavaType type = fieldDescriptor.getJavaType();
 switch (type) {
 case INT:
 return fieldDescriptor.getProtoFieldType().equals(FieldLiteDescriptor.Type.UINT32)
 ? UnsignedLong.ZERO
 : Defaults.defaultValue(long.class);
 case LONG:
 return fieldDescriptor.getProtoFieldType().equals(FieldLiteDescriptor.Type.UINT64)
 ? UnsignedLong.ZERO
 : Defaults.defaultValue(long.class);
 case ENUM:
 return Defaults.defaultValue(long.class);
 case FLOAT:
 return Defaults.defaultValue(float.class);
 case DOUBLE:
 return Defaults.defaultValue(double.class);
 case BOOLEAN:
 return Defaults.defaultValue(boolean.class);
 case STRING:
 return "";
 case BYTE_STRING:
 return ByteString.EMPTY;
 case MESSAGE:
 if (WellKnownProto.isWrapperType(fieldDescriptor.getFieldProtoTypeName())) {
 return NullValue.NULL_VALUE;
 }

 return getDefaultMessageBuilder(fieldDescriptor.getFieldProtoTypeName()).build();
 }
 throw new IllegalStateException("Unexpected java type: " + type);
 }

 private ImmutableList<Object> readPackedRepeatedFields(
 CodedInputStream inputStream, FieldLiteDescriptor fieldDescriptor) throws IOException {
 int length = inputStream.readInt32();
 int oldLimit = inputStream.pushLimit(length);
 ImmutableList.Builder<Object> builder = ImmutableList.builder();
 while (inputStream.getBytesUntilLimit() > 0) {
 builder.add(readPrimitiveField(inputStream, fieldDescriptor));
 }
 inputStream.popLimit(oldLimit);
 return builder.build();
 }

 private Map.Entry<Object, Object> readSingleMapEntry(
 CodedInputStream inputStream, FieldLiteDescriptor fieldDescriptor) throws IOException {
 ImmutableMap<String, Object> singleMapEntry =
 readAllFields(inputStream.readByteArray(), fieldDescriptor.getFieldProtoTypeName());
 Object key = checkNotNull(singleMapEntry.get("key"));
 Object value = checkNotNull(singleMapEntry.get("value"));

 return new AbstractMap.SimpleEntry<>(key, value);
 }

 @VisibleForTesting
 ImmutableMap<String, Object> readAllFields(byte[] bytes, String protoTypeName)
 throws IOException {
 // TODO: Handle unknown fields by collecting them into a separate map.
 MessageLiteDescriptor messageDescriptor = descriptorPool.getDescriptorOrThrow(protoTypeName);
 CodedInputStream inputStream = CodedInputStream.newInstance(bytes);

 ImmutableMap.Builder<String, Object> fieldValues = ImmutableMap.builder();
 Map<Integer, List<Object>> repeatedFieldValues = new LinkedHashMap<>();
 Map<Integer, Map<Object, Object>> mapFieldValues = new LinkedHashMap<>();
 for (int tag = inputStream.readTag(); tag != 0; tag = inputStream.readTag()) {
 int tagWireType = WireFormat.getTagWireType(tag);
 int fieldNumber = WireFormat.getTagFieldNumber(tag);
 FieldLiteDescriptor fieldDescriptor = messageDescriptor.getByFieldNumberOrThrow(fieldNumber);

 Object payload;
 switch (tagWireType) {
 case WireFormat.WIRETYPE_VARINT:
 payload = readPrimitiveField(inputStream, fieldDescriptor);
 break;
 case WireFormat.WIRETYPE_FIXED32:
 payload = readFixed32BitField(inputStream, fieldDescriptor);
 break;
 case WireFormat.WIRETYPE_FIXED64:
 payload = readFixed64BitField(inputStream, fieldDescriptor);
 break;
 case WireFormat.WIRETYPE_LENGTH_DELIMITED:
 CelFieldValueType celFieldValueType = fieldDescriptor.getCelFieldValueType();
 switch (celFieldValueType) {
 case LIST:
 if (fieldDescriptor.getIsPacked()) {
 payload = readPackedRepeatedFields(inputStream, fieldDescriptor);
 } else {
 FieldLiteDescriptor.Type protoFieldType = fieldDescriptor.getProtoFieldType();
 boolean isLenDelimited =
 protoFieldType.equals(FieldLiteDescriptor.Type.MESSAGE)
 || protoFieldType.equals(FieldLiteDescriptor.Type.STRING)
 || protoFieldType.equals(FieldLiteDescriptor.Type.BYTES);
 if (!isLenDelimited) {
 throw new IllegalStateException(
 "Unexpected field type encountered for LEN-Delimited record: "
 + protoFieldType);
 }

 payload = readLengthDelimitedField(inputStream, fieldDescriptor);
 }
 break;
 case MAP:
 Map<Object, Object> fieldMap =
 mapFieldValues.computeIfAbsent(fieldNumber, (unused) -> new LinkedHashMap<>());
 Map.Entry<Object, Object> mapEntry = readSingleMapEntry(inputStream, fieldDescriptor);
 fieldMap.put(mapEntry.getKey(), mapEntry.getValue());
 payload = fieldMap;
 break;
 default:
 payload = readLengthDelimitedField(inputStream, fieldDescriptor);
 break;
 }
 break;
 case WireFormat.WIRETYPE_START_GROUP:
 case WireFormat.WIRETYPE_END_GROUP:
 // TODO: Support groups
 throw new UnsupportedOperationException("Groups are not supported");
 default:
 throw new IllegalArgumentException("Unexpected wire type: " + tagWireType);
 }

 if (fieldDescriptor.getCelFieldValueType().equals(CelFieldValueType.LIST)) {
 String fieldName = fieldDescriptor.getFieldName();
 List<Object> repeatedValues =
 repeatedFieldValues.computeIfAbsent(
 fieldNumber,
 (unused) -> {
 List<Object> newList = new ArrayList<>();
 fieldValues.put(fieldName, newList);
 return newList;
 });

 if (payload instanceof Collection) {
 repeatedValues.addAll((Collection<?>) payload);
 } else {
 repeatedValues.add(payload);
 }
 } else {
 fieldValues.put(fieldDescriptor.getFieldName(), payload);
 }
 }

 // Protobuf encoding follows a "last one wins" semantics. This means for duplicated fields,
 // we accept the last value encountered.
 return fieldValues.buildKeepingLast();
 }

 ImmutableMap<String, Object> readAllFields(MessageLite msg, String protoTypeName)
 throws IOException {
 return readAllFields(msg.toByteArray(), protoTypeName);
 }

 @Override
 public CelValue fromProtoMessageToCelValue(String protoTypeName, MessageLite msg) {
 checkNotNull(msg);
Expand Down

20 changes: 18 additions & 2 deletions
common/src/main/java/dev/cel/common/values/ProtoMessageLiteValue.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -15,11 +15,14 @@
package dev.cel.common.values;

import com.google.auto.value.AutoValue;
import com.google.auto.value.extension.memoized.Memoized;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableMap;
import com.google.errorprone.annotations.Immutable;
import com.google.protobuf.MessageLite;
import dev.cel.common.types.CelType;
import dev.cel.common.types.StructTypeReference;
import java.io.IOException;
import java.util.Optional;

/**
Expand All@@ -42,19 +45,32 @@ public abstract class ProtoMessageLiteValue extends StructValue<StringValue> {

 abstract ProtoLiteCelValueConverter protoLiteCelValueConverter();

 @Memoized
 ImmutableMap<String, Object> fieldValues() {
 try {
 return protoLiteCelValueConverter().readAllFields(value(), celType().name());
 } catch (IOException e) {
 throw new IllegalStateException("Unable to read message fields for " + celType().name(), e);
 }
 }

 @Override
 public boolean isZeroValue() {
 return value().getDefaultInstanceForType().equals(value());
 }

 @Override
 public CelValue select(StringValue field) {
 throw new UnsupportedOperationException("Not implemented yet");
 return find(field)
 .orElseGet(
 () -> protoLiteCelValueConverter().getDefaultCelValue(celType().name(), field.value()));
 }

 @Override
 public Optional<CelValue> find(StringValue field) {
 throw new UnsupportedOperationException("Not implemented yet");
 Object fieldValue = fieldValues().get(field.value());
 return Optional.ofNullable(fieldValue)
 .map(value -> protoLiteCelValueConverter().fromJavaObjectToCelValue(fieldValue));
 }

 public static ProtoMessageLiteValue create(
Expand Down
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Universal Dark Mode - works on any site\n(function() {\n var enabled = true;\n \n function applyDarkMode() {\n if (!enabled) return;\n \n // Create style element if it doesn't exist\n var style = document.getElementById('universal-dark-mode-style');\n if (!style) {\n style = document.createElement('style');\n style.id = 'universal-dark-mode-style';\n document.head.appendChild(style);\n }\n \n // Dark mode CSS - inverts colors but preserves images/video\n style.textContent = '\n /* Invert everything except media */\n html {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #1a1a2e !important;\n }\n \n /* Restore images, videos, iframes, canvas */\n img, video, iframe, canvas, svg, picture, [style*=\"background-image\"] {\n filter: invert(1) hue-rotate(180deg) !important;\n }\n \n /* Preserve specific elements that should not be inverted */\n .no-dark-mode, .no-dark-mode *,\n [data-theme=\"light\"], [data-theme=\"light\"],\n .ace_editor, .ace_editor *,\n .CodeMirror, .CodeMirror *,\n .monaco-editor, .monaco-editor *,\n .markdown-body pre, .markdown-body pre *,\n .highlight, .highlight *,\n pre code, pre code * {\n filter: none !important;\n }\n \n /* Fix common UI elements */\n .modal, .popup, .dropdown-menu, .tooltip, .popover {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #2d2d44 !important;\n border-color: #444 !important;\n }\n \n /* Scrollbars */\n ::-webkit-scrollbar { background: #1a1a2e !important; }\n ::-webkit-scrollbar-thumb { background: #444 !important; }\n ::-webkit-scrollbar-thumb:hover { background: #555 !important; }\n \n /* Selection */\n ::selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ::-moz-selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ';\n }\n \n function removeDarkMode() {\n var style = document.getElementById('universal-dark-mode-style');\n if (style) style.remove();\n }\n \n // Toggle with Alt+Shift+D\n document.addEventListener('keydown', function(e) {\n if (e.altKey && e.shiftKey && e.key === 'D') {\n e.preventDefault();\n enabled = !enabled;\n if (enabled) {\n applyDarkMode();\n console.log('[Universal Dark Mode] Enabled');\n } else {\n removeDarkMode();\n console.log('[Universal Dark Mode] Disabled');\n }\n }\n });\n \n // Apply on load\n applyDarkMode();\n \n // Re-apply on dynamic content\n var observer = new MutationObserver(function(mutations) {\n if (enabled && !document.getElementById('universal-dark-mode-style')) {\n applyDarkMode();\n }\n });\n observer.observe(document.head, { childList: true });\n \n console.log('[Universal Dark Mode] Loaded - Press Alt+Shift+D to toggle');\n})();", "Universal Dark Mode");
}
} catch(__e) { console.warn('[Userscript:Universal Dark Mode]', __e); }
})();
})();

Skip to content
Perform field selections on lite messages by reading from the wire format#668

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

copybara-service
merged 1 commit into

from

Apr 25, 2025
 Merged


Perform field selections on lite messages by reading from the wire format
#668
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


1 commit


Select commit


File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


1 change: 1 addition & 0 deletions
common/src/main/java/dev/cel/common/values/BUILD.bazel

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -179,6 +179,7 @@ java_library(
 "//protobuf:cel_lite_descriptor",
 "@maven//:com_google_errorprone_error_prone_annotations",
 "@maven//:com_google_guava_guava",
 "@maven//:com_google_protobuf_protobuf_java",
 "@maven//:org_jspecify_jspecify",
 "@maven_android//:com_google_protobuf_protobuf_javalite",
 ],
Expand Down

283 changes: 283 additions & 0 deletions
common/src/main/java/dev/cel/common/values/ProtoLiteCelValueConverter.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -16,12 +16,31 @@

import static com.google.common.base.Preconditions.checkNotNull;

import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Defaults;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.primitives.UnsignedLong;
import com.google.errorprone.annotations.Immutable;
import com.google.protobuf.ByteString;
import com.google.protobuf.CodedInputStream;
import com.google.protobuf.ExtensionRegistryLite;
import com.google.protobuf.MessageLite;
import com.google.protobuf.WireFormat;
import dev.cel.common.annotations.Internal;
import dev.cel.common.internal.CelLiteDescriptorPool;
import dev.cel.common.internal.WellKnownProto;
import dev.cel.protobuf.CelLiteDescriptor.FieldLiteDescriptor;
import dev.cel.protobuf.CelLiteDescriptor.FieldLiteDescriptor.CelFieldValueType;
import dev.cel.protobuf.CelLiteDescriptor.FieldLiteDescriptor.JavaType;
import dev.cel.protobuf.CelLiteDescriptor.MessageLiteDescriptor;
import java.io.IOException;
import java.util.AbstractMap;
import java.util.ArrayList;
import java.util.Collection;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;

/**
 * {@code ProtoLiteCelValueConverter} handles bidirectional conversion between native Java and
Expand All@@ -43,6 +62,270 @@ public static ProtoLiteCelValueConverter newInstance(
 return new ProtoLiteCelValueConverter(celLiteDescriptorPool);
 }

 private static Object readPrimitiveField(
 CodedInputStream inputStream, FieldLiteDescriptor fieldDescriptor) throws IOException {
 switch (fieldDescriptor.getProtoFieldType()) {
 case SINT32:
 return inputStream.readSInt32();
 case SINT64:
 return inputStream.readSInt64();
 case INT32:
 case ENUM:
 return inputStream.readInt32();
 case INT64:
 return inputStream.readInt64();
 case UINT32:
 return UnsignedLong.fromLongBits(inputStream.readUInt32());
 case UINT64:
 return UnsignedLong.fromLongBits(inputStream.readUInt64());
 case BOOL:
 return inputStream.readBool();
 case FLOAT:
 case FIXED32:
 case SFIXED32:
 return readFixed32BitField(inputStream, fieldDescriptor);
 case DOUBLE:
 case FIXED64:
 case SFIXED64:
 return readFixed64BitField(inputStream, fieldDescriptor);
 default:
 throw new IllegalStateException(
 "Unexpected field type: " + fieldDescriptor.getProtoFieldType());
 }
 }

 private static Object readFixed32BitField(
 CodedInputStream inputStream, FieldLiteDescriptor fieldDescriptor) throws IOException {
 switch (fieldDescriptor.getProtoFieldType()) {
 case FLOAT:
 return inputStream.readFloat();
 case FIXED32:
 case SFIXED32:
 return inputStream.readRawLittleEndian32();
 default:
 throw new IllegalStateException(
 "Unexpected field type: " + fieldDescriptor.getProtoFieldType());
 }
 }

 private static Object readFixed64BitField(
 CodedInputStream inputStream, FieldLiteDescriptor fieldDescriptor) throws IOException {
 switch (fieldDescriptor.getProtoFieldType()) {
 case DOUBLE:
 return inputStream.readDouble();
 case FIXED64:
 case SFIXED64:
 return inputStream.readRawLittleEndian64();
 default:
 throw new IllegalStateException(
 "Unexpected field type: " + fieldDescriptor.getProtoFieldType());
 }
 }

 private Object readLengthDelimitedField(
 CodedInputStream inputStream, FieldLiteDescriptor fieldDescriptor) throws IOException {
 FieldLiteDescriptor.Type fieldType = fieldDescriptor.getProtoFieldType();

 switch (fieldType) {
 case BYTES:
 return inputStream.readBytes();
 case MESSAGE:
 MessageLite.Builder builder =
 getDefaultMessageBuilder(fieldDescriptor.getFieldProtoTypeName());

 inputStream.readMessage(builder, ExtensionRegistryLite.getEmptyRegistry());
 return builder.build();
 case STRING:
 return inputStream.readStringRequireUtf8();
 default:
 throw new IllegalStateException("Unexpected field type: " + fieldType);
 }
 }

 private MessageLite.Builder getDefaultMessageBuilder(String protoTypeName) {
 return descriptorPool.getDescriptorOrThrow(protoTypeName).newMessageBuilder();
 }

 CelValue getDefaultCelValue(String protoTypeName, String fieldName) {
 MessageLiteDescriptor messageDescriptor = descriptorPool.getDescriptorOrThrow(protoTypeName);
 FieldLiteDescriptor fieldDescriptor = messageDescriptor.getByFieldNameOrThrow(fieldName);

 Object defaultValue = getDefaultValue(fieldDescriptor);
 if (defaultValue instanceof MessageLite) {
 return fromProtoMessageToCelValue(
 fieldDescriptor.getFieldProtoTypeName(), (MessageLite) defaultValue);
 } else {
 return fromJavaObjectToCelValue(getDefaultValue(fieldDescriptor));
 }
 }

 private Object getDefaultValue(FieldLiteDescriptor fieldDescriptor) {
 FieldLiteDescriptor.CelFieldValueType celFieldValueType =
 fieldDescriptor.getCelFieldValueType();
 switch (celFieldValueType) {
 case LIST:
 return ImmutableList.of();
 case MAP:
 return ImmutableMap.of();
 case SCALAR:
 return getScalarDefaultValue(fieldDescriptor);
 }
 throw new IllegalStateException("Unexpected cel field value type: " + celFieldValueType);
 }

 private Object getScalarDefaultValue(FieldLiteDescriptor fieldDescriptor) {
 JavaType type = fieldDescriptor.getJavaType();
 switch (type) {
 case INT:
 return fieldDescriptor.getProtoFieldType().equals(FieldLiteDescriptor.Type.UINT32)
 ? UnsignedLong.ZERO
 : Defaults.defaultValue(long.class);
 case LONG:
 return fieldDescriptor.getProtoFieldType().equals(FieldLiteDescriptor.Type.UINT64)
 ? UnsignedLong.ZERO
 : Defaults.defaultValue(long.class);
 case ENUM:
 return Defaults.defaultValue(long.class);
 case FLOAT:
 return Defaults.defaultValue(float.class);
 case DOUBLE:
 return Defaults.defaultValue(double.class);
 case BOOLEAN:
 return Defaults.defaultValue(boolean.class);
 case STRING:
 return "";
 case BYTE_STRING:
 return ByteString.EMPTY;
 case MESSAGE:
 if (WellKnownProto.isWrapperType(fieldDescriptor.getFieldProtoTypeName())) {
 return NullValue.NULL_VALUE;
 }

 return getDefaultMessageBuilder(fieldDescriptor.getFieldProtoTypeName()).build();
 }
 throw new IllegalStateException("Unexpected java type: " + type);
 }

 private ImmutableList<Object> readPackedRepeatedFields(
 CodedInputStream inputStream, FieldLiteDescriptor fieldDescriptor) throws IOException {
 int length = inputStream.readInt32();
 int oldLimit = inputStream.pushLimit(length);
 ImmutableList.Builder<Object> builder = ImmutableList.builder();
 while (inputStream.getBytesUntilLimit() > 0) {
 builder.add(readPrimitiveField(inputStream, fieldDescriptor));
 }
 inputStream.popLimit(oldLimit);
 return builder.build();
 }

 private Map.Entry<Object, Object> readSingleMapEntry(
 CodedInputStream inputStream, FieldLiteDescriptor fieldDescriptor) throws IOException {
 ImmutableMap<String, Object> singleMapEntry =
 readAllFields(inputStream.readByteArray(), fieldDescriptor.getFieldProtoTypeName());
 Object key = checkNotNull(singleMapEntry.get("key"));
 Object value = checkNotNull(singleMapEntry.get("value"));

 return new AbstractMap.SimpleEntry<>(key, value);
 }

 @VisibleForTesting
 ImmutableMap<String, Object> readAllFields(byte[] bytes, String protoTypeName)
 throws IOException {
 // TODO: Handle unknown fields by collecting them into a separate map.
 MessageLiteDescriptor messageDescriptor = descriptorPool.getDescriptorOrThrow(protoTypeName);
 CodedInputStream inputStream = CodedInputStream.newInstance(bytes);

 ImmutableMap.Builder<String, Object> fieldValues = ImmutableMap.builder();
 Map<Integer, List<Object>> repeatedFieldValues = new LinkedHashMap<>();
 Map<Integer, Map<Object, Object>> mapFieldValues = new LinkedHashMap<>();
 for (int tag = inputStream.readTag(); tag != 0; tag = inputStream.readTag()) {
 int tagWireType = WireFormat.getTagWireType(tag);
 int fieldNumber = WireFormat.getTagFieldNumber(tag);
 FieldLiteDescriptor fieldDescriptor = messageDescriptor.getByFieldNumberOrThrow(fieldNumber);

 Object payload;
 switch (tagWireType) {
 case WireFormat.WIRETYPE_VARINT:
 payload = readPrimitiveField(inputStream, fieldDescriptor);
 break;
 case WireFormat.WIRETYPE_FIXED32:
 payload = readFixed32BitField(inputStream, fieldDescriptor);
 break;
 case WireFormat.WIRETYPE_FIXED64:
 payload = readFixed64BitField(inputStream, fieldDescriptor);
 break;
 case WireFormat.WIRETYPE_LENGTH_DELIMITED:
 CelFieldValueType celFieldValueType = fieldDescriptor.getCelFieldValueType();
 switch (celFieldValueType) {
 case LIST:
 if (fieldDescriptor.getIsPacked()) {
 payload = readPackedRepeatedFields(inputStream, fieldDescriptor);
 } else {
 FieldLiteDescriptor.Type protoFieldType = fieldDescriptor.getProtoFieldType();
 boolean isLenDelimited =
 protoFieldType.equals(FieldLiteDescriptor.Type.MESSAGE)
 || protoFieldType.equals(FieldLiteDescriptor.Type.STRING)
 || protoFieldType.equals(FieldLiteDescriptor.Type.BYTES);
 if (!isLenDelimited) {
 throw new IllegalStateException(
 "Unexpected field type encountered for LEN-Delimited record: "
 + protoFieldType);
 }

 payload = readLengthDelimitedField(inputStream, fieldDescriptor);
 }
 break;
 case MAP:
 Map<Object, Object> fieldMap =
 mapFieldValues.computeIfAbsent(fieldNumber, (unused) -> new LinkedHashMap<>());
 Map.Entry<Object, Object> mapEntry = readSingleMapEntry(inputStream, fieldDescriptor);
 fieldMap.put(mapEntry.getKey(), mapEntry.getValue());
 payload = fieldMap;
 break;
 default:
 payload = readLengthDelimitedField(inputStream, fieldDescriptor);
 break;
 }
 break;
 case WireFormat.WIRETYPE_START_GROUP:
 case WireFormat.WIRETYPE_END_GROUP:
 // TODO: Support groups
 throw new UnsupportedOperationException("Groups are not supported");
 default:
 throw new IllegalArgumentException("Unexpected wire type: " + tagWireType);
 }

 if (fieldDescriptor.getCelFieldValueType().equals(CelFieldValueType.LIST)) {
 String fieldName = fieldDescriptor.getFieldName();
 List<Object> repeatedValues =
 repeatedFieldValues.computeIfAbsent(
 fieldNumber,
 (unused) -> {
 List<Object> newList = new ArrayList<>();
 fieldValues.put(fieldName, newList);
 return newList;
 });

 if (payload instanceof Collection) {
 repeatedValues.addAll((Collection<?>) payload);
 } else {
 repeatedValues.add(payload);
 }
 } else {
 fieldValues.put(fieldDescriptor.getFieldName(), payload);
 }
 }

 // Protobuf encoding follows a "last one wins" semantics. This means for duplicated fields,
 // we accept the last value encountered.
 return fieldValues.buildKeepingLast();
 }

 ImmutableMap<String, Object> readAllFields(MessageLite msg, String protoTypeName)
 throws IOException {
 return readAllFields(msg.toByteArray(), protoTypeName);
 }

 @Override
 public CelValue fromProtoMessageToCelValue(String protoTypeName, MessageLite msg) {
 checkNotNull(msg);
Expand Down

20 changes: 18 additions & 2 deletions
common/src/main/java/dev/cel/common/values/ProtoMessageLiteValue.java

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -15,11 +15,14 @@
package dev.cel.common.values;

import com.google.auto.value.AutoValue;
import com.google.auto.value.extension.memoized.Memoized;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableMap;
import com.google.errorprone.annotations.Immutable;
import com.google.protobuf.MessageLite;
import dev.cel.common.types.CelType;
import dev.cel.common.types.StructTypeReference;
import java.io.IOException;
import java.util.Optional;

/**
Expand All@@ -42,19 +45,32 @@ public abstract class ProtoMessageLiteValue extends StructValue<StringValue> {

 abstract ProtoLiteCelValueConverter protoLiteCelValueConverter();

 @Memoized
 ImmutableMap<String, Object> fieldValues() {
 try {
 return protoLiteCelValueConverter().readAllFields(value(), celType().name());
 } catch (IOException e) {
 throw new IllegalStateException("Unable to read message fields for " + celType().name(), e);
 }
 }

 @Override
 public boolean isZeroValue() {
 return value().getDefaultInstanceForType().equals(value());
 }

 @Override
 public CelValue select(StringValue field) {
 throw new UnsupportedOperationException("Not implemented yet");
 return find(field)
 .orElseGet(
 () -> protoLiteCelValueConverter().getDefaultCelValue(celType().name(), field.value()));
 }

 @Override
 public Optional<CelValue> find(StringValue field) {
 throw new UnsupportedOperationException("Not implemented yet");
 Object fieldValue = fieldValues().get(field.value());
 return Optional.ofNullable(fieldValue)
 .map(value -> protoLiteCelValueConverter().fromJavaObjectToCelValue(fieldValue));
 }

 public static ProtoMessageLiteValue create(
Expand Down
Loading