From 690e0a30a2c65edbc2f2fccb87063afd9e2c4257 Mon Sep 17 00:00:00 2001 From: Javier Rodriguez Date: Mon, 24 Aug 2026 17:02:47 +0200 Subject: [PATCH 1/2] perf(crafter): de-duplicate AccessChk security descriptors in the policy projection The SYSINTERNALS_ACCESSCHK material is projected to JSON client-side for policy evaluation. AccessChk evidence for a registry hive or service database applies a small number of distinct security descriptors to hundreds of thousands of objects through inheritance, and the flat projection repeats each descriptor inline on every object. The policy engine materializes every copy in memory, so a large material drives the CLI to very high peak memory and can OOM-kill the CI runner during evaluation. Project the material to a de-duplicated shape instead: the distinct security descriptors are emitted once in a descriptors table, and each object references one by index. Descriptors are shared only when byte-for-byte identical, so no object, name, or ACE is dropped and policy findings are unchanged. For a large registry material this collapses hundreds of thousands of inline descriptors to a few hundred table entries and brings evaluation peak memory down by an order of magnitude. Policies read a descriptor via input.descriptors[obj.descriptor]. Closes PFM-6922. Assisted-by: Claude Code Signed-off-by: Javier Rodriguez Chainloop-Trace-Sessions: bcafbeb2-90ea-4ff6-89b5-b896e6c68b2f --- .../api/attestation/v1/crafting_state.go | 14 +- .../crafter/materials/accesschk/projection.go | 143 ++++++++++++++++++ .../materials/accesschk/projection_test.go | 95 ++++++++++++ 3 files changed, 251 insertions(+), 1 deletion(-) create mode 100644 pkg/attestation/crafter/materials/accesschk/projection.go create mode 100644 pkg/attestation/crafter/materials/accesschk/projection_test.go diff --git a/pkg/attestation/crafter/api/attestation/v1/crafting_state.go b/pkg/attestation/crafter/api/attestation/v1/crafting_state.go index fa1c5365d..7cc702a14 100644 --- a/pkg/attestation/crafter/api/attestation/v1/crafting_state.go +++ b/pkg/attestation/crafter/api/attestation/v1/crafting_state.go @@ -239,11 +239,23 @@ func (m *Attestation_Material) ingestMaterialToJSON(rawMaterial []byte, value st // AccessChk emits plain text; project it to JSON so the policy engine, // which only consumes JSON, can evaluate it. The raw text is preserved // in the projection's "raw" field for string-matching fallbacks. + // + // The projection de-duplicates security descriptors: a registry hive or + // service database applies a handful of distinct descriptors to hundreds + // of thousands of objects, and repeating each one inline would balloon the + // document the policy engine holds in memory (large materials have + // OOM-killed CI runners). Objects reference a shared descriptors table by + // index instead; no object, name, or ACE is dropped, so policy findings + // are unchanged. Policies read a descriptor via input.descriptors[obj.descriptor]. report, err := accesschk.Parse(bytes.NewReader(rawMaterial)) if err != nil { return nil, fmt.Errorf("invalid accesschk material: %w", err) } - return json.Marshal(report) + projection, err := report.Project() + if err != nil { + return nil, fmt.Errorf("failed to project accesschk material: %w", err) + } + return json.Marshal(projection) case v1.CraftingSchema_Material_CERTCC_DRANZER: // dranzer emits plain text; project it to JSON so the policy engine, // which only consumes JSON, can evaluate it. The raw text is preserved diff --git a/pkg/attestation/crafter/materials/accesschk/projection.go b/pkg/attestation/crafter/materials/accesschk/projection.go new file mode 100644 index 000000000..d1cd99dfc --- /dev/null +++ b/pkg/attestation/crafter/materials/accesschk/projection.go @@ -0,0 +1,143 @@ +// +// Copyright 2026 The Chainloop Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package accesschk + +import ( + "bytes" + "encoding/json" + "reflect" +) + +// Projection is the JSON structure handed to the policy engine at evaluation +// time. It carries exactly the same information as a Report, but the security +// descriptors are de-duplicated: every distinct descriptor (the access-control +// portion of an object) is listed once in Descriptors, and each object references +// one by index. AccessChk evidence for a registry hive or a service database +// applies a handful of distinct descriptors to hundreds of thousands of objects +// through inheritance, so the flat form repeats the same DACL/ACE structures over +// and over. De-duplicating makes the projection — and the value the policy engine +// materialises from it — proportional to the number of DISTINCT descriptors +// rather than the number of objects, without dropping or altering any object, +// name, or ACE, so policy findings are unchanged. +// +// Policies read a descriptor via input.descriptors[obj.descriptor]. See the +// windows-*-strong-acls policies in the compliance-manifests repository. +type Projection struct { + Tool Tool `json:"tool"` + Descriptors []Descriptor `json:"descriptors"` + Objects []ProjectedObject `json:"objects"` + // Raw holds the full original text for inputs below RawRetentionLimit and is + // empty otherwise, mirroring Report.Raw. It is a string-matching fallback and + // is not read by current policies. + Raw string `json:"raw"` +} + +// Descriptor is the security-descriptor portion of an Object: the fields that are +// shared between objects with identical access control. The object name and the +// verbatim raw lines are intentionally excluded — they belong to the object, not +// the descriptor. +type Descriptor struct { + DescriptorFlags []string `json:"descriptor_flags,omitempty"` + Owner string `json:"owner,omitempty"` + DACL []ACE `json:"dacl,omitempty"` + SACL []ACE `json:"sacl,omitempty"` + AccessEntries []AccessEntry `json:"access_entries"` +} + +// ProjectedObject is a securable object in the de-duplicated projection: its name +// and an index into Projection.Descriptors, plus the verbatim RawLines fallback +// when retained (omitted for oversized inputs, matching Report). +type ProjectedObject struct { + Name string `json:"name"` + Descriptor int `json:"descriptor"` + RawLines []string `json:"raw_lines,omitempty"` +} + +// Project converts a parsed Report into its de-duplicated Projection. Two objects +// share a descriptor entry only when their descriptor fields are byte-for-byte +// identical, so no information is lost: every object keeps its own name and its +// exact descriptor, and the mapping is fully reconstructable. +func (r *Report) Project() (*Projection, error) { + p := &Projection{ + Tool: r.Tool, + Descriptors: make([]Descriptor, 0), + Objects: make([]ProjectedObject, 0, len(r.Objects)), + Raw: r.Raw, + } + + // index buckets descriptor positions by a cheap 64-bit fingerprint of their + // canonical JSON. A fingerprint collision is resolved by a full structural + // comparison, so distinct descriptors are never merged (no information lost). + // The serialization reuses a single buffer instead of allocating a fresh key + // per object, which for a hundreds-of-thousands-object material avoids the + // same order of transient garbage as the whole flat projection. + index := make(map[uint64][]int) + var buf bytes.Buffer + enc := json.NewEncoder(&buf) + for i := range r.Objects { + o := &r.Objects[i] + d := Descriptor{ + DescriptorFlags: o.DescriptorFlags, + Owner: o.Owner, + DACL: o.DACL, + SACL: o.SACL, + AccessEntries: o.AccessEntries, + } + + buf.Reset() + if err := enc.Encode(&d); err != nil { + return nil, err + } + fp := fnv64a(buf.Bytes()) + + idx := -1 + for _, cand := range index[fp] { + if reflect.DeepEqual(p.Descriptors[cand], d) { + idx = cand + break + } + } + if idx < 0 { + idx = len(p.Descriptors) + p.Descriptors = append(p.Descriptors, d) + index[fp] = append(index[fp], idx) + } + + p.Objects = append(p.Objects, ProjectedObject{ + Name: o.Name, + Descriptor: idx, + RawLines: o.RawLines, + }) + } + + return p, nil +} + +// fnv64a is the 64-bit FNV-1a hash, inlined to fingerprint a byte slice without +// allocating a hash.Hash. It is used only to bucket candidate descriptors; +// equality is always confirmed structurally, so collisions are harmless. +func fnv64a(b []byte) uint64 { + const ( + offset = 14695981039346656037 + prime = 1099511628211 + ) + h := uint64(offset) + for _, c := range b { + h ^= uint64(c) + h *= prime + } + return h +} diff --git a/pkg/attestation/crafter/materials/accesschk/projection_test.go b/pkg/attestation/crafter/materials/accesschk/projection_test.go new file mode 100644 index 000000000..c15bb3fa5 --- /dev/null +++ b/pkg/attestation/crafter/materials/accesschk/projection_test.go @@ -0,0 +1,95 @@ +// +// Copyright 2026 The Chainloop Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package accesschk + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// Project must be lossless: every object's descriptor is exactly reconstructable +// from the descriptors table, and identical descriptors are shared. +func TestProjectIsLossless(t *testing.T) { + const sys = "SYSTEM" + report := &Report{ + Tool: Tool{Name: ToolName, Version: "6.15"}, + Objects: []Object{ + {Name: "key1", Owner: sys, DACL: []ACE{{Index: 0, Principal: "Everyone", Rights: []string{"KEY_ALL_ACCESS"}}}, AccessEntries: []AccessEntry{}}, + {Name: "key2", Owner: sys, DACL: []ACE{{Index: 0, Principal: "Everyone", Rights: []string{"KEY_ALL_ACCESS"}}}, AccessEntries: []AccessEntry{}}, // same descriptor as key1 + {Name: "key3", Owner: "Admins", DACL: []ACE{{Index: 0, Principal: sys, Rights: []string{"KEY_READ"}}}, AccessEntries: []AccessEntry{}}, // distinct + }, + } + + proj, err := report.Project() + require.NoError(t, err) + + // dedup happened: 3 objects, 2 distinct descriptors. + assert.Len(t, proj.Objects, 3) + assert.Len(t, proj.Descriptors, 2) + assert.Equal(t, proj.Objects[0].Descriptor, proj.Objects[1].Descriptor, "identical descriptors must share an index") + assert.NotEqual(t, proj.Objects[0].Descriptor, proj.Objects[2].Descriptor) + + // every object reconstructs to its original descriptor, byte-for-byte. + for i, po := range proj.Objects { + orig := report.Objects[i] + d := proj.Descriptors[po.Descriptor] + assert.Equal(t, orig.Name, po.Name) + assert.Equal(t, orig.Owner, d.Owner) + assert.Equal(t, orig.DescriptorFlags, d.DescriptorFlags) + assert.Equal(t, orig.DACL, d.DACL) + assert.Equal(t, orig.SACL, d.SACL) + assert.Equal(t, orig.AccessEntries, d.AccessEntries) + } +} + +// A parsed report projected must reconstruct the exact same set of (name, +// descriptor) pairs as the flat objects, over real AccessChk text. +func TestProjectRoundTripFromParse(t *testing.T) { + const sample = `accesschk v6.15 +HKLM\SOFTWARE\A + DESCRIPTOR FLAGS: + [SE_DACL_PRESENT] + OWNER: NT AUTHORITY\SYSTEM + [0] ACCESS_ALLOWED_ACE_TYPE: Everyone + KEY_ALL_ACCESS +HKLM\SOFTWARE\B + DESCRIPTOR FLAGS: + [SE_DACL_PRESENT] + OWNER: NT AUTHORITY\SYSTEM + [0] ACCESS_ALLOWED_ACE_TYPE: Everyone + KEY_ALL_ACCESS +` + report, err := Parse(strings.NewReader(sample)) + require.NoError(t, err) + require.Len(t, report.Objects, 2) + + proj, err := report.Project() + require.NoError(t, err) + require.Len(t, proj.Objects, 2) + // A and B share an identical descriptor. + assert.Len(t, proj.Descriptors, 1) + assert.Equal(t, proj.Objects[0].Descriptor, proj.Objects[1].Descriptor) + + for i, po := range proj.Objects { + d := proj.Descriptors[po.Descriptor] + assert.Equal(t, report.Objects[i].Name, po.Name) + assert.Equal(t, report.Objects[i].DACL, d.DACL) + assert.Equal(t, report.Objects[i].Owner, d.Owner) + } +} From 318c577b556c5b3df1f113b2845f8a138e98eb46 Mon Sep 17 00:00:00 2001 From: Javier Rodriguez Date: Tue, 25 Aug 2026 08:20:10 +0200 Subject: [PATCH 2/2] test(crafter): assert AccessChk projection JSON wire shape Add a test that marshals the projection and verifies the serialized contract the policy engine consumes: the descriptors table, objects referencing a descriptor index (with no inline DACL), the retained/omitted raw_lines fallback, and the top-level raw field. Assisted-by: Claude Code Signed-off-by: Javier Rodriguez Chainloop-Trace-Sessions: bcafbeb2-90ea-4ff6-89b5-b896e6c68b2f --- .../materials/accesschk/projection_test.go | 67 ++++++++++++++++++- 1 file changed, 64 insertions(+), 3 deletions(-) diff --git a/pkg/attestation/crafter/materials/accesschk/projection_test.go b/pkg/attestation/crafter/materials/accesschk/projection_test.go index c15bb3fa5..c96780fb1 100644 --- a/pkg/attestation/crafter/materials/accesschk/projection_test.go +++ b/pkg/attestation/crafter/materials/accesschk/projection_test.go @@ -16,6 +16,7 @@ package accesschk import ( + "encoding/json" "strings" "testing" @@ -23,6 +24,11 @@ import ( "github.com/stretchr/testify/require" ) +const ( + everyone = "Everyone" + keyAllAccess = "KEY_ALL_ACCESS" +) + // Project must be lossless: every object's descriptor is exactly reconstructable // from the descriptors table, and identical descriptors are shared. func TestProjectIsLossless(t *testing.T) { @@ -30,9 +36,9 @@ func TestProjectIsLossless(t *testing.T) { report := &Report{ Tool: Tool{Name: ToolName, Version: "6.15"}, Objects: []Object{ - {Name: "key1", Owner: sys, DACL: []ACE{{Index: 0, Principal: "Everyone", Rights: []string{"KEY_ALL_ACCESS"}}}, AccessEntries: []AccessEntry{}}, - {Name: "key2", Owner: sys, DACL: []ACE{{Index: 0, Principal: "Everyone", Rights: []string{"KEY_ALL_ACCESS"}}}, AccessEntries: []AccessEntry{}}, // same descriptor as key1 - {Name: "key3", Owner: "Admins", DACL: []ACE{{Index: 0, Principal: sys, Rights: []string{"KEY_READ"}}}, AccessEntries: []AccessEntry{}}, // distinct + {Name: "key1", Owner: sys, DACL: []ACE{{Index: 0, Principal: everyone, Rights: []string{keyAllAccess}}}, AccessEntries: []AccessEntry{}}, + {Name: "key2", Owner: sys, DACL: []ACE{{Index: 0, Principal: everyone, Rights: []string{keyAllAccess}}}, AccessEntries: []AccessEntry{}}, // same descriptor as key1 + {Name: "key3", Owner: "Admins", DACL: []ACE{{Index: 0, Principal: sys, Rights: []string{"KEY_READ"}}}, AccessEntries: []AccessEntry{}}, // distinct }, } @@ -93,3 +99,58 @@ HKLM\SOFTWARE\B assert.Equal(t, report.Objects[i].Owner, d.Owner) } } + +// The serialized projection is the contract the policy engine consumes, so assert +// the marshaled JSON shape directly: the descriptors table, objects that carry a +// "descriptor" index (and no inline DACL), the retained/omitted raw_lines +// fallback, and the top-level raw field. +func TestProjectJSONShape(t *testing.T) { + report := &Report{ + Tool: Tool{Name: ToolName, Version: "6.15"}, + Raw: "verbatim text", + Objects: []Object{ + {Name: "key1", DACL: []ACE{{Index: 0, AceType: "access_allowed_ace_type", Principal: everyone, Rights: []string{keyAllAccess}}}, AccessEntries: []AccessEntry{}, RawLines: []string{" raw line"}}, + {Name: "key2", DACL: []ACE{{Index: 0, AceType: "access_allowed_ace_type", Principal: everyone, Rights: []string{keyAllAccess}}}, AccessEntries: []AccessEntry{}}, // same descriptor, no raw lines + }, + } + + proj, err := report.Project() + require.NoError(t, err) + b, err := json.Marshal(proj) + require.NoError(t, err) + + // Decode into a struct mirroring the wire contract policies rely on. + var wire struct { + Tool map[string]any `json:"tool"` + Descriptors []struct { + DACL []map[string]any `json:"dacl"` + AccessEntries []any `json:"access_entries"` + } `json:"descriptors"` + Objects []struct { + Name string `json:"name"` + Descriptor *int `json:"descriptor"` + RawLines []string `json:"raw_lines"` + } `json:"objects"` + Raw string `json:"raw"` + } + require.NoError(t, json.Unmarshal(b, &wire)) + + assert.Equal(t, "verbatim text", wire.Raw) + require.Len(t, wire.Descriptors, 1) // identical descriptors shared + require.Len(t, wire.Objects, 2) + + require.NotNil(t, wire.Objects[0].Descriptor) + assert.Equal(t, "key1", wire.Objects[0].Name) + assert.Equal(t, *wire.Objects[0].Descriptor, *wire.Objects[1].Descriptor) + assert.Equal(t, everyone, wire.Descriptors[*wire.Objects[0].Descriptor].DACL[0]["principal"]) + assert.NotNil(t, wire.Descriptors[0].AccessEntries) + + // raw_lines is retained for key1 and omitted for key2. + assert.Equal(t, []string{" raw line"}, wire.Objects[0].RawLines) + assert.Nil(t, wire.Objects[1].RawLines) + + // Objects reference a descriptor index and never carry the DACL inline. + js := string(b) + assert.Contains(t, js, `"name":"key2","descriptor":0}`) + assert.NotContains(t, js, `"name":"key2","descriptor":0,"dacl"`) +}