perf(crafter): de-duplicate AccessChk security descriptors in the policy projection

pkg/attestation/crafter/api/attestation/v1/crafting_state.go

@@ -239,11 +239,23 @@ func (m *Attestation_Material) ingestMaterialToJSON(rawMaterial []byte, value st
  // AccessChk emits plain text; project it to JSON so the policy engine,
  // which only consumes JSON, can evaluate it. The raw text is preserved
  // in the projection's "raw" field for string-matching fallbacks.
+ //
+ // The projection de-duplicates security descriptors: a registry hive or
+ // service database applies a handful of distinct descriptors to hundreds
+ // of thousands of objects, and repeating each one inline would balloon the
+ // document the policy engine holds in memory (large materials have
+ // OOM-killed CI runners). Objects reference a shared descriptors table by
+ // index instead; no object, name, or ACE is dropped, so policy findings
+ // are unchanged. Policies read a descriptor via input.descriptors[obj.descriptor].
  report, err := accesschk.Parse(bytes.NewReader(rawMaterial))
  if err != nil {
  return nil, fmt.Errorf("invalid accesschk material: %w", err)
  }
- return json.Marshal(report)
+ projection, err := report.Project()
+ if err != nil {
+ return nil, fmt.Errorf("failed to project accesschk material: %w", err)
+ }
+ return json.Marshal(projection)
 	case v1.CraftingSchema_Material_CERTCC_DRANZER:
  // dranzer emits plain text; project it to JSON so the policy engine,
  // which only consumes JSON, can evaluate it. The raw text is preserved

pkg/attestation/crafter/materials/accesschk/projection.go

@@ -0,0 +1,143 @@
+//
+// Copyright 2026 The Chainloop Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package accesschk
+
+import (
+	"bytes"
+	"encoding/json"
+	"reflect"
+)
+
+// Projection is the JSON structure handed to the policy engine at evaluation
+// time. It carries exactly the same information as a Report, but the security
+// descriptors are de-duplicated: every distinct descriptor (the access-control
+// portion of an object) is listed once in Descriptors, and each object references
+// one by index. AccessChk evidence for a registry hive or a service database
+// applies a handful of distinct descriptors to hundreds of thousands of objects
+// through inheritance, so the flat form repeats the same DACL/ACE structures over
+// and over. De-duplicating makes the projection — and the value the policy engine
+// materialises from it — proportional to the number of DISTINCT descriptors
+// rather than the number of objects, without dropping or altering any object,
+// name, or ACE, so policy findings are unchanged.
+//
+// Policies read a descriptor via input.descriptors[obj.descriptor]. See the
+// windows-*-strong-acls policies in the compliance-manifests repository.
+type Projection struct {
+	Tool Tool `json:"tool"`
+	Descriptors []Descriptor `json:"descriptors"`
+	Objects []ProjectedObject `json:"objects"`
+	// Raw holds the full original text for inputs below RawRetentionLimit and is
+	// empty otherwise, mirroring Report.Raw. It is a string-matching fallback and
+	// is not read by current policies.
+	Raw string `json:"raw"`
+}
+
+// Descriptor is the security-descriptor portion of an Object: the fields that are
+// shared between objects with identical access control. The object name and the
+// verbatim raw lines are intentionally excluded — they belong to the object, not
+// the descriptor.
+type Descriptor struct {
+	DescriptorFlags []string `json:"descriptor_flags,omitempty"`
+	Owner string `json:"owner,omitempty"`
+	DACL []ACE `json:"dacl,omitempty"`
+	SACL []ACE `json:"sacl,omitempty"`
+	AccessEntries []AccessEntry `json:"access_entries"`
+}
+
+// ProjectedObject is a securable object in the de-duplicated projection: its name
+// and an index into Projection.Descriptors, plus the verbatim RawLines fallback
+// when retained (omitted for oversized inputs, matching Report).
+type ProjectedObject struct {
+	Name string `json:"name"`
+	Descriptor int `json:"descriptor"`
+	RawLines []string `json:"raw_lines,omitempty"`
+}
+
+// Project converts a parsed Report into its de-duplicated Projection. Two objects
+// share a descriptor entry only when their descriptor fields are byte-for-byte
+// identical, so no information is lost: every object keeps its own name and its
+// exact descriptor, and the mapping is fully reconstructable.
+func (r *Report) Project() (*Projection, error) {
+	p := &Projection{
+ Tool: r.Tool,
+ Descriptors: make([]Descriptor, 0),
+ Objects: make([]ProjectedObject, 0, len(r.Objects)),
+ Raw: r.Raw,
+	}
+
+	// index buckets descriptor positions by a cheap 64-bit fingerprint of their
+	// canonical JSON. A fingerprint collision is resolved by a full structural
+	// comparison, so distinct descriptors are never merged (no information lost).
+	// The serialization reuses a single buffer instead of allocating a fresh key
+	// per object, which for a hundreds-of-thousands-object material avoids the
+	// same order of transient garbage as the whole flat projection.
+	index := make(map[uint64][]int)
+	var buf bytes.Buffer
+	enc := json.NewEncoder(&buf)
+	for i := range r.Objects {
+ o := &r.Objects[i]
+ d := Descriptor{
+ DescriptorFlags: o.DescriptorFlags,
+ Owner: o.Owner,
+ DACL: o.DACL,
+ SACL: o.SACL,
+ AccessEntries: o.AccessEntries,
+ }
+
+ buf.Reset()
+ if err := enc.Encode(&d); err != nil {
+ return nil, err
+ }
+ fp := fnv64a(buf.Bytes())
+
+ idx := -1
+ for _, cand := range index[fp] {
+ if reflect.DeepEqual(p.Descriptors[cand], d) {
+ idx = cand
+ break
+ }
+ }
+ if idx < 0 {
+ idx = len(p.Descriptors)
+ p.Descriptors = append(p.Descriptors, d)
+ index[fp] = append(index[fp], idx)
+ }
+
+ p.Objects = append(p.Objects, ProjectedObject{
+ Name: o.Name,
+ Descriptor: idx,
+ RawLines: o.RawLines,
+ })
+	}
+
+	return p, nil
+}
+
+// fnv64a is the 64-bit FNV-1a hash, inlined to fingerprint a byte slice without
+// allocating a hash.Hash. It is used only to bucket candidate descriptors;
+// equality is always confirmed structurally, so collisions are harmless.
+func fnv64a(b []byte) uint64 {
+	const (
+ offset = 14695981039346656037
+ prime = 1099511628211
+	)
+	h := uint64(offset)
+	for _, c := range b {
+ h ^= uint64(c)
+ h *= prime
+	}
+	return h
+}

pkg/attestation/crafter/materials/accesschk/projection_test.go

@@ -0,0 +1,156 @@
+//
+// Copyright 2026 The Chainloop Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package accesschk
+
+import (
+	"encoding/json"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+const (
+	everyone = "Everyone"
+	keyAllAccess = "KEY_ALL_ACCESS"
+)
+
+// Project must be lossless: every object's descriptor is exactly reconstructable
+// from the descriptors table, and identical descriptors are shared.
+func TestProjectIsLossless(t *testing.T) {
+	const sys = "SYSTEM"
+	report := &Report{
+ Tool: Tool{Name: ToolName, Version: "6.15"},
+ Objects: []Object{
+ {Name: "key1", Owner: sys, DACL: []ACE{{Index: 0, Principal: everyone, Rights: []string{keyAllAccess}}}, AccessEntries: []AccessEntry{}},
+ {Name: "key2", Owner: sys, DACL: []ACE{{Index: 0, Principal: everyone, Rights: []string{keyAllAccess}}}, AccessEntries: []AccessEntry{}}, // same descriptor as key1
+ {Name: "key3", Owner: "Admins", DACL: []ACE{{Index: 0, Principal: sys, Rights: []string{"KEY_READ"}}}, AccessEntries: []AccessEntry{}}, // distinct
+ },
+	}
+
+	proj, err := report.Project()
+	require.NoError(t, err)
+
+	// dedup happened: 3 objects, 2 distinct descriptors.
+	assert.Len(t, proj.Objects, 3)
+	assert.Len(t, proj.Descriptors, 2)
+	assert.Equal(t, proj.Objects[0].Descriptor, proj.Objects[1].Descriptor, "identical descriptors must share an index")
+	assert.NotEqual(t, proj.Objects[0].Descriptor, proj.Objects[2].Descriptor)
+
+	// every object reconstructs to its original descriptor, byte-for-byte.
+	for i, po := range proj.Objects {
+ orig := report.Objects[i]
+ d := proj.Descriptors[po.Descriptor]
+ assert.Equal(t, orig.Name, po.Name)
+ assert.Equal(t, orig.Owner, d.Owner)
+ assert.Equal(t, orig.DescriptorFlags, d.DescriptorFlags)
+ assert.Equal(t, orig.DACL, d.DACL)
+ assert.Equal(t, orig.SACL, d.SACL)
+ assert.Equal(t, orig.AccessEntries, d.AccessEntries)
+	}
+}
+
+// A parsed report projected must reconstruct the exact same set of (name,
+// descriptor) pairs as the flat objects, over real AccessChk text.
+func TestProjectRoundTripFromParse(t *testing.T) {
+	const sample = `accesschk v6.15
+HKLM\SOFTWARE\A
+ DESCRIPTOR FLAGS:
+ [SE_DACL_PRESENT]
+ OWNER: NT AUTHORITY\SYSTEM
+ [0] ACCESS_ALLOWED_ACE_TYPE: Everyone
+	KEY_ALL_ACCESS
+HKLM\SOFTWARE\B
+ DESCRIPTOR FLAGS:
+ [SE_DACL_PRESENT]
+ OWNER: NT AUTHORITY\SYSTEM
+ [0] ACCESS_ALLOWED_ACE_TYPE: Everyone
+	KEY_ALL_ACCESS
+`
+	report, err := Parse(strings.NewReader(sample))
+	require.NoError(t, err)
+	require.Len(t, report.Objects, 2)
+
+	proj, err := report.Project()
+	require.NoError(t, err)
+	require.Len(t, proj.Objects, 2)
+	// A and B share an identical descriptor.
+	assert.Len(t, proj.Descriptors, 1)
+	assert.Equal(t, proj.Objects[0].Descriptor, proj.Objects[1].Descriptor)
+
+	for i, po := range proj.Objects {
+ d := proj.Descriptors[po.Descriptor]
+ assert.Equal(t, report.Objects[i].Name, po.Name)
+ assert.Equal(t, report.Objects[i].DACL, d.DACL)
+ assert.Equal(t, report.Objects[i].Owner, d.Owner)
+	}
+}
+
+// The serialized projection is the contract the policy engine consumes, so assert
+// the marshaled JSON shape directly: the descriptors table, objects that carry a
+// "descriptor" index (and no inline DACL), the retained/omitted raw_lines
+// fallback, and the top-level raw field.
+func TestProjectJSONShape(t *testing.T) {
+	report := &Report{
+ Tool: Tool{Name: ToolName, Version: "6.15"},
+ Raw: "verbatim text",
+ Objects: []Object{
+ {Name: "key1", DACL: []ACE{{Index: 0, AceType: "access_allowed_ace_type", Principal: everyone, Rights: []string{keyAllAccess}}}, AccessEntries: []AccessEntry{}, RawLines: []string{" raw line"}},
+ {Name: "key2", DACL: []ACE{{Index: 0, AceType: "access_allowed_ace_type", Principal: everyone, Rights: []string{keyAllAccess}}}, AccessEntries: []AccessEntry{}}, // same descriptor, no raw lines
+ },
+	}
+
+	proj, err := report.Project()
+	require.NoError(t, err)
+	b, err := json.Marshal(proj)
+	require.NoError(t, err)
+
+	// Decode into a struct mirroring the wire contract policies rely on.
+	var wire struct {
+ Tool map[string]any `json:"tool"`
+ Descriptors []struct {
+ DACL []map[string]any `json:"dacl"`
+ AccessEntries []any `json:"access_entries"`
+ } `json:"descriptors"`
+ Objects []struct {
+ Name string `json:"name"`
+ Descriptor *int `json:"descriptor"`
+ RawLines []string `json:"raw_lines"`
+ } `json:"objects"`
+ Raw string `json:"raw"`
+	}
+	require.NoError(t, json.Unmarshal(b, &wire))
+
+	assert.Equal(t, "verbatim text", wire.Raw)
+	require.Len(t, wire.Descriptors, 1) // identical descriptors shared
+	require.Len(t, wire.Objects, 2)
+
+	require.NotNil(t, wire.Objects[0].Descriptor)
+	assert.Equal(t, "key1", wire.Objects[0].Name)
+	assert.Equal(t, *wire.Objects[0].Descriptor, *wire.Objects[1].Descriptor)
+	assert.Equal(t, everyone, wire.Descriptors[*wire.Objects[0].Descriptor].DACL[0]["principal"])
+	assert.NotNil(t, wire.Descriptors[0].AccessEntries)
+
+	// raw_lines is retained for key1 and omitted for key2.
+	assert.Equal(t, []string{" raw line"}, wire.Objects[0].RawLines)
+	assert.Nil(t, wire.Objects[1].RawLines)
+
+	// Objects reference a descriptor index and never carry the DACL inline.
+	js := string(b)
+	assert.Contains(t, js, `"name":"key2","descriptor":0}`)
+	assert.NotContains(t, js, `"name":"key2","descriptor":0,"dacl"`)
+}