From 31512b3bc93fe4549509d23cab433b1b1aa3f84b Mon Sep 17 00:00:00 2001 From: Antoine Toussaint Date: Fri, 21 Aug 2026 19:49:56 -0400 Subject: [PATCH 1/4] feat(librarystore): versioned library publishing store with GitHub Go strategy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit First step of the durable library-publishing epic (#446): a backend-agnostic Store that publishes and resolves library exports as versioned artifacts a native package manager can consume without the codefly toolchain. - Store interface (Publish/Resolve/List) over Coordinates{language,name,version} → Published{importPath, immutable ref, digest, install hint}. - GitHubStore Go strategy: publishes each Go export to its own repo tagged vX.Y.Z, so `go get github.com//-go@vX.Y.Z` resolves natively. Published versions are immutable (republish is refused); Resolve selects the highest tag satisfying a semver constraint and pins the tag's commit. - Python/TypeScript return an explicit not-implemented error (P1). Tested end-to-end against a real local bare git repository (publish → list → resolve → immutability), no mocks. Part of #446. Co-Authored-By: Claude Opus 4.8 --- pkg/librarystore/github.go | 301 ++++++++++++++++++++++++++++++++ pkg/librarystore/github_test.go | 115 ++++++++++++ pkg/librarystore/store.go | 56 ++++++ 3 files changed, 472 insertions(+) create mode 100644 pkg/librarystore/github.go create mode 100644 pkg/librarystore/github_test.go create mode 100644 pkg/librarystore/store.go diff --git a/pkg/librarystore/github.go b/pkg/librarystore/github.go new file mode 100644 index 00000000..e554ea65 --- /dev/null +++ b/pkg/librarystore/github.go @@ -0,0 +1,301 @@ +package librarystore + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "io/fs" + "os" + "os/exec" + "path/filepath" + "sort" + "strings" + + "github.com/Masterminds/semver" +) + +// GitHubStore publishes each library export to its own GitHub repository, tagged +// with a semantic version. A Go export published this way is resolvable by +// `go get github.com//-go@vX.Y.Z` with no codefly toolchain, since +// a git repository with semver tags is a native Go module source. +type GitHubStore struct { + // Owner is the GitHub organization or user the published repositories live in. + Owner string + // remoteFor resolves the git remote for a library export. Tests override it to + // point at a local bare repository so publish/resolve exercise real git. + remoteFor func(language Language, name string) string + // commitIdentity is the author used for release commits. + commitName string + commitEmail string +} + +// NewGitHubStore returns a store publishing to repositories under owner. +func NewGitHubStore(owner string) *GitHubStore { + s := &GitHubStore{Owner: owner, commitName: "codefly", commitEmail: "bot@codefly.dev"} + s.remoteFor = func(language Language, name string) string { + return fmt.Sprintf("https://github.com/%s/%s.git", owner, repositoryName(language, name)) + } + return s +} + +// repositoryName is the per-export repository name, e.g. "authkit-go". +func repositoryName(language Language, name string) string { + return fmt.Sprintf("%s-%s", name, language) +} + +// goModulePath derives the Go module path from a remote URL: the identity a +// consumer passes to `go get`. It is only meaningful for real remote URLs; a +// local test remote yields a path used purely for assertions. +func goModulePath(remote string) string { + path := strings.TrimSuffix(remote, ".git") + path = strings.TrimPrefix(path, "https://") + path = strings.TrimPrefix(path, "git@") + path = strings.Replace(path, ":", "/", 1) + return path +} + +func versionTag(version string) string { + return "v" + strings.TrimPrefix(version, "v") +} + +func (s *GitHubStore) Publish(ctx context.Context, artifactDir string, c Coordinates) (Published, error) { + if c.Language != LanguageGo { + return Published{}, fmt.Errorf("librarystore: publishing %s libraries is not implemented yet", c.Language) + } + if _, err := semver.NewVersion(strings.TrimPrefix(c.Version, "v")); err != nil { + return Published{}, fmt.Errorf("librarystore: %q is not a semantic version: %w", c.Version, err) + } + remote := s.remoteFor(c.Language, c.Name) + tag := versionTag(c.Version) + + work, err := os.MkdirTemp("", "codefly-library-publish-*") + if err != nil { + return Published{}, err + } + defer os.RemoveAll(work) + + if err = s.git(ctx, "", "clone", "--quiet", remote, work); err != nil { + return Published{}, fmt.Errorf("clone %s: %w", remote, err) + } + if s.tagExists(ctx, work, tag) { + return Published{}, fmt.Errorf("librarystore: %s %s is already published (versions are immutable)", c.Name, tag) + } + // A fresh library repository is an unborn branch; an existing one already has + // main. checkout -b creates it on the first publish, checkout switches to it + // afterwards. + if err = s.git(ctx, work, "checkout", "main"); err != nil { + if createErr := s.git(ctx, work, "checkout", "-b", "main"); createErr != nil { + return Published{}, fmt.Errorf("prepare main branch: %w", createErr) + } + } + if err = replaceTrackedTree(work, artifactDir); err != nil { + return Published{}, fmt.Errorf("stage artifact: %w", err) + } + if err = s.git(ctx, work, "add", "-A"); err != nil { + return Published{}, err + } + message := fmt.Sprintf("release %s %s", c.Name, tag) + if err = s.git(ctx, work, + "-c", "user.name="+s.commitName, "-c", "user.email="+s.commitEmail, + "commit", "--quiet", "-m", message); err != nil { + return Published{}, fmt.Errorf("commit release: %w", err) + } + if err = s.git(ctx, work, + "-c", "user.name="+s.commitName, "-c", "user.email="+s.commitEmail, + "tag", "-a", tag, "-m", message); err != nil { + return Published{}, fmt.Errorf("tag release: %w", err) + } + if err = s.git(ctx, work, "push", "--quiet", "origin", "main"); err != nil { + return Published{}, fmt.Errorf("push branch: %w", err) + } + if err = s.git(ctx, work, "push", "--quiet", "origin", tag); err != nil { + return Published{}, fmt.Errorf("push tag: %w", err) + } + commit, err := s.output(ctx, work, "rev-parse", tag+"^{commit}") + if err != nil { + return Published{}, err + } + digest, err := treeDigest(artifactDir) + if err != nil { + return Published{}, err + } + return s.published(c, remote, strings.TrimSpace(commit), digest), nil +} + +func (s *GitHubStore) Resolve(ctx context.Context, language Language, name, constraint string) (Published, error) { + if language != LanguageGo { + return Published{}, fmt.Errorf("librarystore: resolving %s libraries is not implemented yet", language) + } + versions, err := s.List(ctx, language, name) + if err != nil { + return Published{}, err + } + if len(versions) == 0 { + return Published{}, fmt.Errorf("librarystore: no published versions of %s", name) + } + check, err := semver.NewConstraint(strings.TrimSpace(constraint)) + if err != nil { + return Published{}, fmt.Errorf("librarystore: invalid version constraint %q: %w", constraint, err) + } + var best *semver.Version + for _, candidate := range versions { + v, parseErr := semver.NewVersion(candidate) + if parseErr != nil { + continue + } + if check.Check(v) && (best == nil || v.GreaterThan(best)) { + best = v + } + } + if best == nil { + return Published{}, fmt.Errorf("librarystore: no published version of %s satisfies %q", name, constraint) + } + remote := s.remoteFor(language, name) + commit, err := s.output(ctx, "", "ls-remote", remote, "refs/tags/"+versionTag(best.String())+"^{}") + if err != nil { + return Published{}, err + } + ref := strings.TrimSpace(strings.SplitN(strings.TrimSpace(commit), "\t", 2)[0]) + return s.published(Coordinates{Language: language, Name: name, Version: best.String()}, remote, ref, ""), nil +} + +func (s *GitHubStore) List(ctx context.Context, language Language, name string) ([]string, error) { + if language != LanguageGo { + return nil, fmt.Errorf("librarystore: listing %s libraries is not implemented yet", language) + } + remote := s.remoteFor(language, name) + out, err := s.output(ctx, "", "ls-remote", "--tags", remote) + if err != nil { + return nil, err + } + var versions []*semver.Version + for _, line := range strings.Split(strings.TrimSpace(out), "\n") { + fields := strings.Fields(line) + if len(fields) != 2 { + continue + } + ref := strings.TrimSuffix(fields[1], "^{}") + tag := strings.TrimPrefix(ref, "refs/tags/") + if tag == ref || !strings.HasPrefix(tag, "v") { + continue + } + v, err := semver.NewVersion(strings.TrimPrefix(tag, "v")) + if err != nil { + continue + } + versions = append(versions, v) + } + sort.Slice(versions, func(i, j int) bool { return versions[i].GreaterThan(versions[j]) }) + seen := map[string]struct{}{} + result := make([]string, 0, len(versions)) + for _, v := range versions { + key := v.String() + if _, ok := seen[key]; ok { + continue + } + seen[key] = struct{}{} + result = append(result, key) + } + return result, nil +} + +func (s *GitHubStore) published(c Coordinates, remote, ref, digest string) Published { + importPath := goModulePath(remote) + tag := versionTag(c.Version) + return Published{ + Coordinates: c, + ImportPath: importPath, + Ref: ref, + Location: remote, + Digest: digest, + InstallHint: fmt.Sprintf("go get %s@%s", importPath, tag), + } +} + +func (s *GitHubStore) tagExists(ctx context.Context, dir, tag string) bool { + return s.git(ctx, dir, "rev-parse", "--verify", "--quiet", "refs/tags/"+tag) == nil +} + +func (s *GitHubStore) git(ctx context.Context, dir string, args ...string) error { + //nolint:gosec // git is invoked with internal subcommands and store-controlled arguments, never a shell. + command := exec.CommandContext(ctx, "git", gitArgs(dir, args)...) + if output, err := command.CombinedOutput(); err != nil { + return fmt.Errorf("git %s: %w: %s", strings.Join(args, " "), err, strings.TrimSpace(string(output))) + } + return nil +} + +func (s *GitHubStore) output(ctx context.Context, dir string, args ...string) (string, error) { + //nolint:gosec // git is invoked with internal subcommands and store-controlled arguments, never a shell. + command := exec.CommandContext(ctx, "git", gitArgs(dir, args)...) + out, err := command.Output() + if err != nil { + return "", fmt.Errorf("git %s: %w", strings.Join(args, " "), err) + } + return string(out), nil +} + +func gitArgs(dir string, args []string) []string { + if dir == "" { + return args + } + return append([]string{"-C", dir}, args...) +} + +// replaceTrackedTree makes the working tree's tracked content equal source: it +// removes every entry except .git, then copies source in. This keeps a release +// commit an exact snapshot of the packaged export, dropping files a prior +// version had and this one does not. os.CopyFS copies regular files and +// directories and rejects symlinks, so a published export is self-contained. +func replaceTrackedTree(work, source string) error { + entries, err := os.ReadDir(work) + if err != nil { + return err + } + for _, entry := range entries { + if entry.Name() == ".git" { + continue + } + if err := os.RemoveAll(filepath.Join(work, entry.Name())); err != nil { + return err + } + } + return os.CopyFS(work, os.DirFS(source)) +} + +// treeDigest is a deterministic sha256 over the regular files under root, keyed +// by slash-separated relative path, so an identical export yields an identical +// digest. It reads through a root-scoped filesystem, so a symlink cannot escape +// root during the walk. +func treeDigest(root string) (string, error) { + type entry struct { + path string + digest [32]byte + } + fsys := os.DirFS(root) + var entries []entry + err := fs.WalkDir(fsys, ".", func(path string, dirEntry fs.DirEntry, err error) error { + if err != nil { + return err + } + if dirEntry.IsDir() || !dirEntry.Type().IsRegular() { + return nil + } + data, err := fs.ReadFile(fsys, path) + if err != nil { + return err + } + entries = append(entries, entry{path: path, digest: sha256.Sum256(data)}) + return nil + }) + if err != nil { + return "", err + } + sort.Slice(entries, func(i, j int) bool { return entries[i].path < entries[j].path }) + hasher := sha256.New() + for _, e := range entries { + fmt.Fprintf(hasher, "%s\x00%s\n", e.path, hex.EncodeToString(e.digest[:])) + } + return "sha256:" + hex.EncodeToString(hasher.Sum(nil)), nil +} diff --git a/pkg/librarystore/github_test.go b/pkg/librarystore/github_test.go new file mode 100644 index 00000000..8df19b8f --- /dev/null +++ b/pkg/librarystore/github_test.go @@ -0,0 +1,115 @@ +package librarystore + +import ( + "context" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +func bareRepo(t *testing.T) string { + t.Helper() + dir := filepath.Join(t.TempDir(), "remote.git") + require.NoError(t, exec.Command("git", "init", "--quiet", "--bare", dir).Run()) + return dir +} + +func goModule(t *testing.T, modulePath, body string) string { + t.Helper() + dir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(dir, "go.mod"), []byte("module "+modulePath+"\n\ngo 1.26\n"), 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "lib.go"), []byte(body), 0o644)) + return dir +} + +func storeTo(remote string) *GitHubStore { + s := NewGitHubStore("codefly-dev") + s.remoteFor = func(Language, string) string { return remote } + return s +} + +func TestGitHubStorePublishResolveGoLibrary(t *testing.T) { + ctx := context.Background() + remote := bareRepo(t) + store := storeTo(remote) + + first, err := store.Publish(ctx, goModule(t, "example.com/authkit", "package authkit\n\nconst V = 1\n"), + Coordinates{Language: LanguageGo, Name: "authkit", Version: "1.0.0"}) + require.NoError(t, err) + require.Equal(t, "authkit", first.Name) + require.Equal(t, goModulePath(remote), first.ImportPath) + require.Contains(t, first.InstallHint, "@v1.0.0") + require.True(t, strings.HasPrefix(first.Digest, "sha256:")) + require.Len(t, first.Ref, 40) + + // A second version publishes and lists newest-first. + _, err = store.Publish(ctx, goModule(t, "example.com/authkit", "package authkit\n\nconst V = 2\n"), + Coordinates{Language: LanguageGo, Name: "authkit", Version: "1.2.0"}) + require.NoError(t, err) + + versions, err := store.List(ctx, LanguageGo, "authkit") + require.NoError(t, err) + require.Equal(t, []string{"1.2.0", "1.0.0"}, versions) + + // A constraint resolves to the highest satisfying published version. + resolved, err := store.Resolve(ctx, LanguageGo, "authkit", "^1.0.0") + require.NoError(t, err) + require.Equal(t, "1.2.0", resolved.Version) + require.Len(t, resolved.Ref, 40) + require.Contains(t, resolved.InstallHint, "@v1.2.0") +} + +func TestGitHubStorePublishedVersionsAreImmutable(t *testing.T) { + ctx := context.Background() + store := storeTo(bareRepo(t)) + coords := Coordinates{Language: LanguageGo, Name: "authkit", Version: "1.0.0"} + + _, err := store.Publish(ctx, goModule(t, "example.com/authkit", "package authkit\n"), coords) + require.NoError(t, err) + + _, err = store.Publish(ctx, goModule(t, "example.com/authkit", "package authkit // changed\n"), coords) + require.Error(t, err) + require.Contains(t, err.Error(), "already published") +} + +func TestGitHubStoreRejectsNonSemverAndUnsupportedLanguages(t *testing.T) { + ctx := context.Background() + store := storeTo(bareRepo(t)) + + _, err := store.Publish(ctx, t.TempDir(), Coordinates{Language: LanguageGo, Name: "authkit", Version: "latest"}) + require.Error(t, err) + + _, err = store.Publish(ctx, t.TempDir(), Coordinates{Language: LanguagePython, Name: "authkit", Version: "1.0.0"}) + require.ErrorContains(t, err, "not implemented") +} + +func TestGoModulePathAndRepositoryName(t *testing.T) { + require.Equal(t, "authkit-go", repositoryName(LanguageGo, "authkit")) + require.Equal(t, "github.com/codefly-dev/authkit-go", goModulePath("https://github.com/codefly-dev/authkit-go.git")) + require.Equal(t, "github.com/codefly-dev/authkit-go", goModulePath("git@github.com:codefly-dev/authkit-go.git")) +} + +func TestDefaultRemoteFor(t *testing.T) { + store := NewGitHubStore("codefly-dev") + require.Equal(t, "https://github.com/codefly-dev/authkit-go.git", store.remoteFor(LanguageGo, "authkit")) +} + +func TestTreeDigestIsDeterministicAndContentSensitive(t *testing.T) { + a := goModule(t, "example.com/x", "package x\n") + b := goModule(t, "example.com/x", "package x\n") + c := goModule(t, "example.com/x", "package x // different\n") + + da, err := treeDigest(a) + require.NoError(t, err) + db, err := treeDigest(b) + require.NoError(t, err) + dc, err := treeDigest(c) + require.NoError(t, err) + + require.Equal(t, da, db) + require.NotEqual(t, da, dc) +} diff --git a/pkg/librarystore/store.go b/pkg/librarystore/store.go new file mode 100644 index 00000000..3e197b75 --- /dev/null +++ b/pkg/librarystore/store.go @@ -0,0 +1,56 @@ +// Package librarystore publishes and resolves codefly libraries as durable, +// versioned artifacts consumable by a language's native package manager +// (go get / pip / npm), so a consumer needs neither the codefly toolchain nor +// local source. The Store interface is backend-agnostic; the GitHub-backed +// implementation is the first (and, for now, only) backend. +package librarystore + +import "context" + +// Language identifies a library's language export. +type Language string + +const ( + LanguageGo Language = "go" + LanguagePython Language = "python" + LanguageTypeScript Language = "typescript" +) + +// Coordinates identify one language export of a library at a semantic version. +type Coordinates struct { + Language Language + Name string + Version string // semantic version, with or without a leading "v" +} + +// Published is a resolved, durable location for a library export that a native +// package manager can consume. +type Published struct { + Coordinates + // ImportPath is the identity the native tool uses: a Go module path, a pip + // distribution name, or an npm @scope/name. + ImportPath string + // Ref is the immutable anchor the version resolves to — a git commit — so a + // moved tag is detectable. + Ref string + // Location is the backing URL. + Location string + // Digest is the sha256 over the published artifact tree ("sha256:"). + Digest string + // InstallHint is a copy-pasteable native install command. + InstallHint string +} + +// Store publishes and resolves library exports through some backend. +type Store interface { + // Publish uploads the artifact tree at artifactDir as the given coordinates + // and returns the durable, resolvable location. Publishing a version that + // already exists is an error: published versions are immutable. + Publish(ctx context.Context, artifactDir string, c Coordinates) (Published, error) + // Resolve selects the highest published version satisfying constraint and + // returns its durable location. + Resolve(ctx context.Context, language Language, name, constraint string) (Published, error) + // List returns the published semantic versions for a library export, without + // a leading "v", newest first. + List(ctx context.Context, language Language, name string) ([]string, error) +} From 3928516ad7d544ce23fe6bac5dfbd823fb1be621 Mon Sep 17 00:00:00 2001 From: Antoine Toussaint Date: Fri, 21 Aug 2026 20:18:42 -0400 Subject: [PATCH 2/4] fix(librarystore): validate the go get promise at publish time, plus review fixes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review of the store found the product promise itself was unverified, and four ways real inputs break the publish/resolve path. Fix the causes: - Module-path validation: `go get @` only works when the published go.mod declares exactly that module path, and versions are immutable — so a mismatched release could never be corrected. Publish now requires go.mod to declare the derived import path, failing before any git operation. The e2e test previously enshrined the broken case (published example.com/authkit into an authkit-go repo); it now publishes the matching path, and a dedicated test proves rejection fires pre-clone. - Source .git collision: replaceTrackedTree's os.CopyFS rewrite dropped the .git skip, so publishing a library that is itself a git checkout would interleave its .git into the clone's .git before failing. Restore the skip (root-scoped walk, symlinks still rejected). - Identical-content releases: a version bump with no code change staged an empty diff and `git commit` failed. Commit with --allow-empty. - Silent empty Ref: Resolve read the tag commit from a second ls-remote whose peeled `^{}` line does not exist for lightweight tags, yielding Ref:"" with no error. One ls-remote now feeds both List and Resolve (peeled commit wins, lightweight hash falls back, no listing/lookup race) and the ref is validated as a commit hash. - Digest parity and modes: Resolve returned Digest:"" though Published documents it; it now computes the digest from a shallow clone at the tag. treeDigest folds in the git-normalized mode (755/644) so an executable-bit flip changes the digest — the same lesson the docker-build-recipe v2 contract encoded — while staying identical between the publish-side source and a resolve-side clone; .git is excluded on both sides. - Publish honors the repository's default branch (symbolic-ref) instead of forcing main; a missing remote repo now errors with create-it-first guidance; Published.Version is canonicalized (no leading v) on both paths; prerelease exclusion under plain constraints is pinned by test. Co-Authored-By: Claude Fable 5 --- pkg/librarystore/github.go | 266 ++++++++++++++++++++++++++------ pkg/librarystore/github_test.go | 170 ++++++++++++++++++-- pkg/librarystore/store.go | 9 +- 3 files changed, 379 insertions(+), 66 deletions(-) diff --git a/pkg/librarystore/github.go b/pkg/librarystore/github.go index e554ea65..61cd58ab 100644 --- a/pkg/librarystore/github.go +++ b/pkg/librarystore/github.go @@ -15,6 +15,10 @@ import ( "github.com/Masterminds/semver" ) +// gitDir is the directory git owns in a working tree. It is never part of +// published content: skipped when staging a source tree and when digesting. +const gitDir = ".git" + // GitHubStore publishes each library export to its own GitHub repository, tagged // with a semantic version. A Go export published this way is resolvable by // `go get github.com//-go@vX.Y.Z` with no codefly toolchain, since @@ -63,12 +67,27 @@ func (s *GitHubStore) Publish(ctx context.Context, artifactDir string, c Coordin if c.Language != LanguageGo { return Published{}, fmt.Errorf("librarystore: publishing %s libraries is not implemented yet", c.Language) } - if _, err := semver.NewVersion(strings.TrimPrefix(c.Version, "v")); err != nil { + version, err := semver.NewVersion(strings.TrimPrefix(c.Version, "v")) + if err != nil { return Published{}, fmt.Errorf("librarystore: %q is not a semantic version: %w", c.Version, err) } + c.Version = version.String() remote := s.remoteFor(c.Language, c.Name) + importPath := goModulePath(remote) tag := versionTag(c.Version) + // The product promise is that consumers run `go get @`. That + // only works when the published go.mod declares exactly that module path, so a + // mismatch must fail here, at publish time — after the tag lands the version + // is immutable and a broken release could never be corrected. + if err = validateGoModulePath(artifactDir, importPath); err != nil { + return Published{}, err + } + digest, err := treeDigest(artifactDir) + if err != nil { + return Published{}, err + } + work, err := os.MkdirTemp("", "codefly-library-publish-*") if err != nil { return Published{}, err @@ -76,18 +95,14 @@ func (s *GitHubStore) Publish(ctx context.Context, artifactDir string, c Coordin defer os.RemoveAll(work) if err = s.git(ctx, "", "clone", "--quiet", remote, work); err != nil { - return Published{}, fmt.Errorf("clone %s: %w", remote, err) + return Published{}, fmt.Errorf("clone %s (create the library repository first if it does not exist yet): %w", remote, err) } if s.tagExists(ctx, work, tag) { return Published{}, fmt.Errorf("librarystore: %s %s is already published (versions are immutable)", c.Name, tag) } - // A fresh library repository is an unborn branch; an existing one already has - // main. checkout -b creates it on the first publish, checkout switches to it - // afterwards. - if err = s.git(ctx, work, "checkout", "main"); err != nil { - if createErr := s.git(ctx, work, "checkout", "-b", "main"); createErr != nil { - return Published{}, fmt.Errorf("prepare main branch: %w", createErr) - } + branch, err := s.defaultBranch(ctx, work) + if err != nil { + return Published{}, err } if err = replaceTrackedTree(work, artifactDir); err != nil { return Published{}, fmt.Errorf("stage artifact: %w", err) @@ -96,9 +111,11 @@ func (s *GitHubStore) Publish(ctx context.Context, artifactDir string, c Coordin return Published{}, err } message := fmt.Sprintf("release %s %s", c.Name, tag) + // --allow-empty: a release whose content is identical to the previous one + // (a version bump with no code change) is a valid release, not an error. if err = s.git(ctx, work, "-c", "user.name="+s.commitName, "-c", "user.email="+s.commitEmail, - "commit", "--quiet", "-m", message); err != nil { + "commit", "--quiet", "--allow-empty", "-m", message); err != nil { return Published{}, fmt.Errorf("commit release: %w", err) } if err = s.git(ctx, work, @@ -106,7 +123,7 @@ func (s *GitHubStore) Publish(ctx context.Context, artifactDir string, c Coordin "tag", "-a", tag, "-m", message); err != nil { return Published{}, fmt.Errorf("tag release: %w", err) } - if err = s.git(ctx, work, "push", "--quiet", "origin", "main"); err != nil { + if err = s.git(ctx, work, "push", "--quiet", "origin", branch); err != nil { return Published{}, fmt.Errorf("push branch: %w", err) } if err = s.git(ctx, work, "push", "--quiet", "origin", tag); err != nil { @@ -116,66 +133,132 @@ func (s *GitHubStore) Publish(ctx context.Context, artifactDir string, c Coordin if err != nil { return Published{}, err } - digest, err := treeDigest(artifactDir) + return s.published(c, remote, strings.TrimSpace(commit), digest), nil +} + +// validateGoModulePath requires the artifact's go.mod to declare importPath as +// its module path — the identity consumers will `go get`. +func validateGoModulePath(artifactDir, importPath string) error { + data, err := os.ReadFile(filepath.Join(artifactDir, "go.mod")) if err != nil { - return Published{}, err + return fmt.Errorf("librarystore: a Go library export must contain a go.mod: %w", err) } - return s.published(c, remote, strings.TrimSpace(commit), digest), nil + declared, err := parseGoModulePath(data) + if err != nil { + return fmt.Errorf("librarystore: invalid go.mod: %w", err) + } + if declared != importPath { + return fmt.Errorf( + "librarystore: go.mod declares module %q but consumers will require %q; the export must be rewritten to the published module path before publishing", + declared, + importPath, + ) + } + return nil +} + +func parseGoModulePath(data []byte) (string, error) { + for _, line := range strings.Split(string(data), "\n") { + if index := strings.Index(line, "//"); index >= 0 { + line = line[:index] + } + fields := strings.Fields(line) + if len(fields) >= 2 && fields[0] == "module" { + return strings.Trim(fields[1], `"`), nil + } + } + return "", fmt.Errorf("no module directive") } func (s *GitHubStore) Resolve(ctx context.Context, language Language, name, constraint string) (Published, error) { if language != LanguageGo { return Published{}, fmt.Errorf("librarystore: resolving %s libraries is not implemented yet", language) } - versions, err := s.List(ctx, language, name) + remote := s.remoteFor(language, name) + tagged, err := s.listTagged(ctx, remote) if err != nil { return Published{}, err } - if len(versions) == 0 { + if len(tagged) == 0 { return Published{}, fmt.Errorf("librarystore: no published versions of %s", name) } check, err := semver.NewConstraint(strings.TrimSpace(constraint)) if err != nil { return Published{}, fmt.Errorf("librarystore: invalid version constraint %q: %w", constraint, err) } - var best *semver.Version - for _, candidate := range versions { - v, parseErr := semver.NewVersion(candidate) - if parseErr != nil { - continue - } - if check.Check(v) && (best == nil || v.GreaterThan(best)) { - best = v + var best *taggedVersion + for index := range tagged { + candidate := &tagged[index] + if check.Check(candidate.version) && (best == nil || candidate.version.GreaterThan(best.version)) { + best = candidate } } if best == nil { return Published{}, fmt.Errorf("librarystore: no published version of %s satisfies %q", name, constraint) } - remote := s.remoteFor(language, name) - commit, err := s.output(ctx, "", "ls-remote", remote, "refs/tags/"+versionTag(best.String())+"^{}") + if !isCommitHash(best.ref) { + return Published{}, fmt.Errorf("librarystore: tag %s of %s resolves to invalid commit %q", versionTag(best.version.String()), name, best.ref) + } + digest, err := s.digestAtTag(ctx, remote, versionTag(best.version.String())) if err != nil { return Published{}, err } - ref := strings.TrimSpace(strings.SplitN(strings.TrimSpace(commit), "\t", 2)[0]) - return s.published(Coordinates{Language: language, Name: name, Version: best.String()}, remote, ref, ""), nil + coordinates := Coordinates{Language: language, Name: name, Version: best.version.String()} + return s.published(coordinates, remote, best.ref, digest), nil } func (s *GitHubStore) List(ctx context.Context, language Language, name string) ([]string, error) { if language != LanguageGo { return nil, fmt.Errorf("librarystore: listing %s libraries is not implemented yet", language) } - remote := s.remoteFor(language, name) + tagged, err := s.listTagged(ctx, s.remoteFor(language, name)) + if err != nil { + return nil, err + } + result := make([]string, 0, len(tagged)) + for _, entry := range tagged { + result = append(result, entry.version.String()) + } + return result, nil +} + +// taggedVersion is one published semver tag with the commit it resolves to. +type taggedVersion struct { + version *semver.Version + ref string +} + +// listTagged reads the remote's semver tags and their commits in one ls-remote, +// newest first. Capturing the commit here — instead of a second ls-remote at +// resolve time — removes the window in which the tag set could change between +// listing and commit lookup. +func (s *GitHubStore) listTagged(ctx context.Context, remote string) ([]taggedVersion, error) { out, err := s.output(ctx, "", "ls-remote", "--tags", remote) if err != nil { return nil, err } - var versions []*semver.Version + return parseTagListing(out), nil +} + +// parseTagListing extracts semver tags from `ls-remote --tags` output. An +// annotated tag appears twice — the tag object and the peeled `^{}` commit — +// and the peeled commit wins; a lightweight tag appears once and its hash is +// already the commit. +func parseTagListing(out string) []taggedVersion { + type entry struct { + version *semver.Version + ref string + peeled bool + } + byVersion := map[string]entry{} for _, line := range strings.Split(strings.TrimSpace(out), "\n") { fields := strings.Fields(line) if len(fields) != 2 { continue } - ref := strings.TrimSuffix(fields[1], "^{}") + ref := fields[1] + peeled := strings.HasSuffix(ref, "^{}") + ref = strings.TrimSuffix(ref, "^{}") tag := strings.TrimPrefix(ref, "refs/tags/") if tag == ref || !strings.HasPrefix(tag, "v") { continue @@ -184,20 +267,41 @@ func (s *GitHubStore) List(ctx context.Context, language Language, name string) if err != nil { continue } - versions = append(versions, v) - } - sort.Slice(versions, func(i, j int) bool { return versions[i].GreaterThan(versions[j]) }) - seen := map[string]struct{}{} - result := make([]string, 0, len(versions)) - for _, v := range versions { key := v.String() - if _, ok := seen[key]; ok { + if existing, ok := byVersion[key]; ok && existing.peeled && !peeled { continue } - seen[key] = struct{}{} - result = append(result, key) + byVersion[key] = entry{version: v, ref: fields[0], peeled: peeled} } - return result, nil + result := make([]taggedVersion, 0, len(byVersion)) + for _, e := range byVersion { + result = append(result, taggedVersion{version: e.version, ref: e.ref}) + } + sort.Slice(result, func(i, j int) bool { return result[i].version.GreaterThan(result[j].version) }) + return result +} + +func isCommitHash(value string) bool { + if len(value) != 40 && len(value) != 64 { + return false + } + _, err := hex.DecodeString(value) + return err == nil +} + +// digestAtTag computes the content digest of a published version from a shallow +// clone at its tag, so Resolve honors the Published digest contract with the +// same value Publish recorded. +func (s *GitHubStore) digestAtTag(ctx context.Context, remote, tag string) (string, error) { + work, err := os.MkdirTemp("", "codefly-library-resolve-*") + if err != nil { + return "", err + } + defer os.RemoveAll(work) + if err := s.git(ctx, "", "clone", "--quiet", "--depth", "1", "--branch", tag, remote, work); err != nil { + return "", fmt.Errorf("fetch %s at %s: %w", remote, tag, err) + } + return treeDigest(work) } func (s *GitHubStore) published(c Coordinates, remote, ref, digest string) Published { @@ -217,6 +321,18 @@ func (s *GitHubStore) tagExists(ctx context.Context, dir, tag string) bool { return s.git(ctx, dir, "rev-parse", "--verify", "--quiet", "refs/tags/"+tag) == nil } +// defaultBranch reports the branch the clone checked out — the remote's default +// branch, or the unborn initial branch of an empty repository. Publishing on it +// (rather than forcing "main") keeps the published repository's branch layout +// matching its GitHub default. +func (s *GitHubStore) defaultBranch(ctx context.Context, work string) (string, error) { + out, err := s.output(ctx, work, "symbolic-ref", "--short", "HEAD") + if err != nil { + return "", fmt.Errorf("resolve default branch: %w", err) + } + return strings.TrimSpace(out), nil +} + func (s *GitHubStore) git(ctx context.Context, dir string, args ...string) error { //nolint:gosec // git is invoked with internal subcommands and store-controlled arguments, never a shell. command := exec.CommandContext(ctx, "git", gitArgs(dir, args)...) @@ -246,31 +362,65 @@ func gitArgs(dir string, args []string) []string { // replaceTrackedTree makes the working tree's tracked content equal source: it // removes every entry except .git, then copies source in. This keeps a release // commit an exact snapshot of the packaged export, dropping files a prior -// version had and this one does not. os.CopyFS copies regular files and -// directories and rejects symlinks, so a published export is self-contained. +// version had and this one does not. A .git directory inside source (a library +// that is itself a git checkout) is skipped — copying it would interleave with +// and corrupt the clone's own .git — and symlinks are rejected so a published +// export is self-contained. func replaceTrackedTree(work, source string) error { entries, err := os.ReadDir(work) if err != nil { return err } for _, entry := range entries { - if entry.Name() == ".git" { + if entry.Name() == gitDir { continue } if err := os.RemoveAll(filepath.Join(work, entry.Name())); err != nil { return err } } - return os.CopyFS(work, os.DirFS(source)) + fsys := os.DirFS(source) + return fs.WalkDir(fsys, ".", func(path string, entry fs.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + if path == "." { + return nil + } + target := filepath.Join(work, filepath.FromSlash(path)) + if entry.IsDir() { + if entry.Name() == gitDir { + return fs.SkipDir + } + return os.MkdirAll(target, 0o755) + } + if !entry.Type().IsRegular() { + return fmt.Errorf("%s: only regular files and directories may be published", path) + } + data, err := fs.ReadFile(fsys, path) + if err != nil { + return err + } + info, err := entry.Info() + if err != nil { + return err + } + return os.WriteFile(target, data, info.Mode().Perm()) + }) } // treeDigest is a deterministic sha256 over the regular files under root, keyed -// by slash-separated relative path, so an identical export yields an identical -// digest. It reads through a root-scoped filesystem, so a symlink cannot escape -// root during the walk. +// by slash-separated relative path and the git-normalized file mode. Git +// preserves only the executable bit through publish and clone, so the digest +// folds each mode to 755/644 — capturing an executable-bit flip (a real change, +// per the docker-build-recipe v2 contract) while staying identical between the +// publish-side source tree and a resolve-side clone. Any .git directory is +// excluded: it is never part of the published content. Reads go through a +// root-scoped filesystem, so a symlink cannot escape root during the walk. func treeDigest(root string) (string, error) { type entry struct { path string + mode string digest [32]byte } fsys := os.DirFS(root) @@ -279,14 +429,28 @@ func treeDigest(root string) (string, error) { if err != nil { return err } - if dirEntry.IsDir() || !dirEntry.Type().IsRegular() { + if dirEntry.IsDir() { + if dirEntry.Name() == gitDir { + return fs.SkipDir + } + return nil + } + if !dirEntry.Type().IsRegular() { return nil } data, err := fs.ReadFile(fsys, path) if err != nil { return err } - entries = append(entries, entry{path: path, digest: sha256.Sum256(data)}) + info, err := dirEntry.Info() + if err != nil { + return err + } + mode := "644" + if info.Mode()&0o111 != 0 { + mode = "755" + } + entries = append(entries, entry{path: path, mode: mode, digest: sha256.Sum256(data)}) return nil }) if err != nil { @@ -295,7 +459,7 @@ func treeDigest(root string) (string, error) { sort.Slice(entries, func(i, j int) bool { return entries[i].path < entries[j].path }) hasher := sha256.New() for _, e := range entries { - fmt.Fprintf(hasher, "%s\x00%s\n", e.path, hex.EncodeToString(e.digest[:])) + fmt.Fprintf(hasher, "%s\x00%s\x00%s\n", e.path, e.mode, hex.EncodeToString(e.digest[:])) } return "sha256:" + hex.EncodeToString(hasher.Sum(nil)), nil } diff --git a/pkg/librarystore/github_test.go b/pkg/librarystore/github_test.go index 8df19b8f..239c5a1c 100644 --- a/pkg/librarystore/github_test.go +++ b/pkg/librarystore/github_test.go @@ -36,44 +36,101 @@ func TestGitHubStorePublishResolveGoLibrary(t *testing.T) { ctx := context.Background() remote := bareRepo(t) store := storeTo(remote) + modulePath := goModulePath(remote) - first, err := store.Publish(ctx, goModule(t, "example.com/authkit", "package authkit\n\nconst V = 1\n"), + first, err := store.Publish(ctx, goModule(t, modulePath, "package authkit\n\nconst V = 1\n"), Coordinates{Language: LanguageGo, Name: "authkit", Version: "1.0.0"}) require.NoError(t, err) require.Equal(t, "authkit", first.Name) - require.Equal(t, goModulePath(remote), first.ImportPath) + require.Equal(t, modulePath, first.ImportPath) require.Contains(t, first.InstallHint, "@v1.0.0") require.True(t, strings.HasPrefix(first.Digest, "sha256:")) require.Len(t, first.Ref, 40) - // A second version publishes and lists newest-first. - _, err = store.Publish(ctx, goModule(t, "example.com/authkit", "package authkit\n\nconst V = 2\n"), - Coordinates{Language: LanguageGo, Name: "authkit", Version: "1.2.0"}) + // A second version publishes and lists newest-first; a "v"-prefixed input + // version is normalized to canonical form. + second, err := store.Publish(ctx, goModule(t, modulePath, "package authkit\n\nconst V = 2\n"), + Coordinates{Language: LanguageGo, Name: "authkit", Version: "v1.2.0"}) require.NoError(t, err) + require.Equal(t, "1.2.0", second.Version) versions, err := store.List(ctx, LanguageGo, "authkit") require.NoError(t, err) require.Equal(t, []string{"1.2.0", "1.0.0"}, versions) - // A constraint resolves to the highest satisfying published version. + // A constraint resolves to the highest satisfying published version, with the + // commit pinned and the digest matching what Publish recorded. resolved, err := store.Resolve(ctx, LanguageGo, "authkit", "^1.0.0") require.NoError(t, err) require.Equal(t, "1.2.0", resolved.Version) - require.Len(t, resolved.Ref, 40) + require.Equal(t, second.Ref, resolved.Ref) + require.Equal(t, second.Digest, resolved.Digest) require.Contains(t, resolved.InstallHint, "@v1.2.0") } -func TestGitHubStorePublishedVersionsAreImmutable(t *testing.T) { +func TestGitHubStorePublishedVersionsAreImmutableButIdenticalContentReleases(t *testing.T) { ctx := context.Background() - store := storeTo(bareRepo(t)) + remote := bareRepo(t) + store := storeTo(remote) + modulePath := goModulePath(remote) coords := Coordinates{Language: LanguageGo, Name: "authkit", Version: "1.0.0"} - _, err := store.Publish(ctx, goModule(t, "example.com/authkit", "package authkit\n"), coords) + _, err := store.Publish(ctx, goModule(t, modulePath, "package authkit\n"), coords) require.NoError(t, err) - _, err = store.Publish(ctx, goModule(t, "example.com/authkit", "package authkit // changed\n"), coords) + // Republishing an existing version is refused regardless of content. + _, err = store.Publish(ctx, goModule(t, modulePath, "package authkit // changed\n"), coords) require.Error(t, err) require.Contains(t, err.Error(), "already published") + + // A new version whose content is byte-identical to the previous release is a + // valid release, not an empty-commit failure. + identical, err := store.Publish(ctx, goModule(t, modulePath, "package authkit\n"), + Coordinates{Language: LanguageGo, Name: "authkit", Version: "1.0.1"}) + require.NoError(t, err) + require.Equal(t, "1.0.1", identical.Version) +} + +func TestGitHubStoreResolveExcludesPrereleases(t *testing.T) { + ctx := context.Background() + remote := bareRepo(t) + store := storeTo(remote) + modulePath := goModulePath(remote) + + _, err := store.Publish(ctx, goModule(t, modulePath, "package authkit\n"), + Coordinates{Language: LanguageGo, Name: "authkit", Version: "1.0.0"}) + require.NoError(t, err) + _, err = store.Publish(ctx, goModule(t, modulePath, "package authkit // rc\n"), + Coordinates{Language: LanguageGo, Name: "authkit", Version: "1.1.0-rc1"}) + require.NoError(t, err) + + versions, err := store.List(ctx, LanguageGo, "authkit") + require.NoError(t, err) + require.Equal(t, []string{"1.1.0-rc1", "1.0.0"}, versions) + + // A plain caret constraint never selects a prerelease. + resolved, err := store.Resolve(ctx, LanguageGo, "authkit", "^1.0.0") + require.NoError(t, err) + require.Equal(t, "1.0.0", resolved.Version) +} + +func TestGitHubStoreRejectsModulePathMismatchBeforeTouchingTheRemote(t *testing.T) { + ctx := context.Background() + // An unreachable remote proves validation fires before any git operation: a + // clone attempt would fail with a network error, not a module-path error. + store := storeTo("https://192.0.2.1/unreachable/authkit-go.git") + + _, err := store.Publish(ctx, goModule(t, "example.com/wrong", "package authkit\n"), + Coordinates{Language: LanguageGo, Name: "authkit", Version: "1.0.0"}) + require.Error(t, err) + require.Contains(t, err.Error(), "declares module") + require.Contains(t, err.Error(), "example.com/wrong") + + // A missing go.mod is rejected the same way. + _, err = store.Publish(ctx, t.TempDir(), + Coordinates{Language: LanguageGo, Name: "authkit", Version: "1.0.0"}) + require.Error(t, err) + require.Contains(t, err.Error(), "go.mod") } func TestGitHubStoreRejectsNonSemverAndUnsupportedLanguages(t *testing.T) { @@ -87,6 +144,61 @@ func TestGitHubStoreRejectsNonSemverAndUnsupportedLanguages(t *testing.T) { require.ErrorContains(t, err, "not implemented") } +func TestReplaceTrackedTreeSkipsSourceGitAndPreservesCloneGit(t *testing.T) { + work := t.TempDir() + require.NoError(t, os.MkdirAll(filepath.Join(work, ".git"), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(work, ".git", "HEAD"), []byte("ref: refs/heads/main\n"), 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(work, "stale.go"), []byte("old"), 0o644)) + + source := t.TempDir() + require.NoError(t, os.MkdirAll(filepath.Join(source, ".git", "refs", "heads"), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(source, ".git", "config"), []byte("[core]\n"), 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(source, "lib.go"), []byte("package lib\n"), 0o644)) + + require.NoError(t, replaceTrackedTree(work, source)) + + // The clone's git state is untouched, the source's .git never copied, prior + // content dropped, and new content in place. + head, err := os.ReadFile(filepath.Join(work, ".git", "HEAD")) + require.NoError(t, err) + require.Equal(t, "ref: refs/heads/main\n", string(head)) + _, err = os.Stat(filepath.Join(work, ".git", "config")) + require.True(t, os.IsNotExist(err), "source .git must not be copied into the clone") + _, err = os.Stat(filepath.Join(work, "stale.go")) + require.True(t, os.IsNotExist(err)) + data, err := os.ReadFile(filepath.Join(work, "lib.go")) + require.NoError(t, err) + require.Equal(t, "package lib\n", string(data)) +} + +func TestParseTagListingPeeledWinsAndLightweightFallsBack(t *testing.T) { + annotatedObject := strings.Repeat("a", 40) + annotatedCommit := strings.Repeat("b", 40) + lightweightCommit := strings.Repeat("c", 40) + out := annotatedObject + "\trefs/tags/v1.0.0\n" + + annotatedCommit + "\trefs/tags/v1.0.0^{}\n" + + lightweightCommit + "\trefs/tags/v2.0.0\n" + + strings.Repeat("d", 40) + "\trefs/heads/main\n" + + strings.Repeat("e", 40) + "\trefs/tags/not-semver\n" + + tagged := parseTagListing(out) + require.Len(t, tagged, 2) + // Newest first; a lightweight tag's hash is already the commit. + require.Equal(t, "2.0.0", tagged[0].version.String()) + require.Equal(t, lightweightCommit, tagged[0].ref) + // The peeled commit wins over the annotated tag object. + require.Equal(t, "1.0.0", tagged[1].version.String()) + require.Equal(t, annotatedCommit, tagged[1].ref) +} + +func TestIsCommitHash(t *testing.T) { + require.True(t, isCommitHash(strings.Repeat("a", 40))) + require.True(t, isCommitHash(strings.Repeat("0", 64))) + require.False(t, isCommitHash("")) + require.False(t, isCommitHash("v1.0.0")) + require.False(t, isCommitHash(strings.Repeat("g", 40))) +} + func TestGoModulePathAndRepositoryName(t *testing.T) { require.Equal(t, "authkit-go", repositoryName(LanguageGo, "authkit")) require.Equal(t, "github.com/codefly-dev/authkit-go", goModulePath("https://github.com/codefly-dev/authkit-go.git")) @@ -98,7 +210,20 @@ func TestDefaultRemoteFor(t *testing.T) { require.Equal(t, "https://github.com/codefly-dev/authkit-go.git", store.remoteFor(LanguageGo, "authkit")) } -func TestTreeDigestIsDeterministicAndContentSensitive(t *testing.T) { +func TestParseGoModulePath(t *testing.T) { + path, err := parseGoModulePath([]byte("// release\nmodule github.com/x/y // comment\n\ngo 1.26\n")) + require.NoError(t, err) + require.Equal(t, "github.com/x/y", path) + + path, err = parseGoModulePath([]byte(`module "github.com/x/quoted"` + "\n")) + require.NoError(t, err) + require.Equal(t, "github.com/x/quoted", path) + + _, err = parseGoModulePath([]byte("go 1.26\n")) + require.Error(t, err) +} + +func TestTreeDigestDeterministicContentAndModeSensitiveGitExcluded(t *testing.T) { a := goModule(t, "example.com/x", "package x\n") b := goModule(t, "example.com/x", "package x\n") c := goModule(t, "example.com/x", "package x // different\n") @@ -109,7 +234,26 @@ func TestTreeDigestIsDeterministicAndContentSensitive(t *testing.T) { require.NoError(t, err) dc, err := treeDigest(c) require.NoError(t, err) - require.Equal(t, da, db) require.NotEqual(t, da, dc) + + // Flipping the executable bit is a real change — git preserves it into the + // published tree. + require.NoError(t, os.Chmod(filepath.Join(b, "lib.go"), 0o755)) + dbExec, err := treeDigest(b) + require.NoError(t, err) + require.NotEqual(t, da, dbExec) + + // A non-executable mode variation git cannot preserve does not change it. + require.NoError(t, os.Chmod(filepath.Join(a, "lib.go"), 0o640)) + daTightened, err := treeDigest(a) + require.NoError(t, err) + require.Equal(t, da, daTightened) + + // .git content is never part of the published tree. + require.NoError(t, os.MkdirAll(filepath.Join(a, ".git"), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(a, ".git", "config"), []byte("[core]\n"), 0o644)) + daWithGit, err := treeDigest(a) + require.NoError(t, err) + require.Equal(t, da, daWithGit) } diff --git a/pkg/librarystore/store.go b/pkg/librarystore/store.go index 3e197b75..ad0d60b0 100644 --- a/pkg/librarystore/store.go +++ b/pkg/librarystore/store.go @@ -20,7 +20,9 @@ const ( type Coordinates struct { Language Language Name string - Version string // semantic version, with or without a leading "v" + // Version is a semantic version. Callers may pass it with or without a + // leading "v"; every Published carries it in canonical form without one. + Version string } // Published is a resolved, durable location for a library export that a native @@ -35,7 +37,10 @@ type Published struct { Ref string // Location is the backing URL. Location string - // Digest is the sha256 over the published artifact tree ("sha256:"). + // Digest is the sha256 over the published artifact tree ("sha256:"), + // set by Publish from the source tree and by Resolve from the published + // content at the resolved version. The two agree for a store-published + // version. Digest string // InstallHint is a copy-pasteable native install command. InstallHint string From 264f68a49aa7db9490062dff1c73ade4b0c89cf0 Mon Sep 17 00:00:00 2001 From: Antoine Toussaint Date: Fri, 21 Aug 2026 20:34:52 -0400 Subject: [PATCH 3/4] fix(librarystore): harden publish env, digest domain, and owner config MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow-up review of the store surfaced three remaining defects: - Digest domain mismatch: Publish hashed the raw source dir while Resolve hashed a clone, so a .gitignore'd file made the two disagree — breaking the Published "the two agree" contract. treeDigest now hashes the git-tracked file set (`git ls-files`) on both paths, so it describes exactly the published content and Publish/Resolve always match. - Non-hermetic git: commit/tag inherited ambient config, so a global commit.gpgsign=true (or a push credential prompt) could fail or hang a real publish. Signing is now disabled per-invocation and git runs with GIT_TERMINAL_PROMPT=0, while push credentials are preserved. - Owner foot-gun: the default remoteFor closed over the constructor argument, so reassigning GitHubStore.Owner silently published to the original owner. It now reads s.Owner live. Tests: digest ignores untracked/ignored files and tracks the exec bit; Publish succeeds under a global config that forces signing with a bogus gpg program; Owner reassignment retargets the remote. Co-Authored-By: Claude Opus 4.8 --- pkg/librarystore/github.go | 86 ++++++++++++++------------- pkg/librarystore/github_test.go | 102 +++++++++++++++++++++++++------- 2 files changed, 125 insertions(+), 63 deletions(-) diff --git a/pkg/librarystore/github.go b/pkg/librarystore/github.go index 61cd58ab..b80636a8 100644 --- a/pkg/librarystore/github.go +++ b/pkg/librarystore/github.go @@ -38,7 +38,7 @@ type GitHubStore struct { func NewGitHubStore(owner string) *GitHubStore { s := &GitHubStore{Owner: owner, commitName: "codefly", commitEmail: "bot@codefly.dev"} s.remoteFor = func(language Language, name string) string { - return fmt.Sprintf("https://github.com/%s/%s.git", owner, repositoryName(language, name)) + return fmt.Sprintf("https://github.com/%s/%s.git", s.Owner, repositoryName(language, name)) } return s } @@ -83,10 +83,6 @@ func (s *GitHubStore) Publish(ctx context.Context, artifactDir string, c Coordin if err = validateGoModulePath(artifactDir, importPath); err != nil { return Published{}, err } - digest, err := treeDigest(artifactDir) - if err != nil { - return Published{}, err - } work, err := os.MkdirTemp("", "codefly-library-publish-*") if err != nil { @@ -115,11 +111,13 @@ func (s *GitHubStore) Publish(ctx context.Context, artifactDir string, c Coordin // (a version bump with no code change) is a valid release, not an error. if err = s.git(ctx, work, "-c", "user.name="+s.commitName, "-c", "user.email="+s.commitEmail, + "-c", "commit.gpgsign=false", "commit", "--quiet", "--allow-empty", "-m", message); err != nil { return Published{}, fmt.Errorf("commit release: %w", err) } if err = s.git(ctx, work, "-c", "user.name="+s.commitName, "-c", "user.email="+s.commitEmail, + "-c", "tag.gpgsign=false", "tag", "-a", tag, "-m", message); err != nil { return Published{}, fmt.Errorf("tag release: %w", err) } @@ -133,6 +131,10 @@ func (s *GitHubStore) Publish(ctx context.Context, artifactDir string, c Coordin if err != nil { return Published{}, err } + digest, err := s.treeDigest(ctx, work) + if err != nil { + return Published{}, err + } return s.published(c, remote, strings.TrimSpace(commit), digest), nil } @@ -301,7 +303,7 @@ func (s *GitHubStore) digestAtTag(ctx context.Context, remote, tag string) (stri if err := s.git(ctx, "", "clone", "--quiet", "--depth", "1", "--branch", tag, remote, work); err != nil { return "", fmt.Errorf("fetch %s at %s: %w", remote, tag, err) } - return treeDigest(work) + return s.treeDigest(ctx, work) } func (s *GitHubStore) published(c Coordinates, remote, ref, digest string) Published { @@ -336,6 +338,7 @@ func (s *GitHubStore) defaultBranch(ctx context.Context, work string) (string, e func (s *GitHubStore) git(ctx context.Context, dir string, args ...string) error { //nolint:gosec // git is invoked with internal subcommands and store-controlled arguments, never a shell. command := exec.CommandContext(ctx, "git", gitArgs(dir, args)...) + command.Env = gitEnv() if output, err := command.CombinedOutput(); err != nil { return fmt.Errorf("git %s: %w: %s", strings.Join(args, " "), err, strings.TrimSpace(string(output))) } @@ -345,6 +348,7 @@ func (s *GitHubStore) git(ctx context.Context, dir string, args ...string) error func (s *GitHubStore) output(ctx context.Context, dir string, args ...string) (string, error) { //nolint:gosec // git is invoked with internal subcommands and store-controlled arguments, never a shell. command := exec.CommandContext(ctx, "git", gitArgs(dir, args)...) + command.Env = gitEnv() out, err := command.Output() if err != nil { return "", fmt.Errorf("git %s: %w", strings.Join(args, " "), err) @@ -352,6 +356,15 @@ func (s *GitHubStore) output(ctx context.Context, dir string, args ...string) (s return string(out), nil } +// gitEnv runs git non-interactively: a missing credential fails fast rather than +// blocking on a terminal prompt, and no credential-manager UI is launched. +// Ambient configuration is otherwise preserved so a user's configured push +// credentials still work; signing is disabled per-invocation on the commands +// that create objects, not by discarding global config wholesale. +func gitEnv() []string { + return append(os.Environ(), "GIT_TERMINAL_PROMPT=0", "GCM_INTERACTIVE=Never") +} + func gitArgs(dir string, args []string) []string { if dir == "" { return args @@ -409,52 +422,45 @@ func replaceTrackedTree(work, source string) error { }) } -// treeDigest is a deterministic sha256 over the regular files under root, keyed -// by slash-separated relative path and the git-normalized file mode. Git -// preserves only the executable bit through publish and clone, so the digest -// folds each mode to 755/644 — capturing an executable-bit flip (a real change, -// per the docker-build-recipe v2 contract) while staying identical between the -// publish-side source tree and a resolve-side clone. Any .git directory is -// excluded: it is never part of the published content. Reads go through a -// root-scoped filesystem, so a symlink cannot escape root during the walk. -func treeDigest(root string) (string, error) { +// treeDigest is a deterministic sha256 over the git-tracked files in dir, keyed +// by slash-separated relative path and git-normalized mode. Driving the file +// set from `git ls-files` — rather than a filesystem walk — means the digest +// describes exactly the published content: files a .gitignore excludes are +// never committed and never hashed, so the publish-side working tree and a +// resolve-side clone of the same version produce the same digest. Git preserves +// only the executable bit, so modes fold to 755/644 — capturing an +// executable-bit flip (a real change, per the docker-build-recipe v2 contract) +// while ignoring mode variations git itself does not carry. +func (s *GitHubStore) treeDigest(ctx context.Context, dir string) (string, error) { + out, err := s.output(ctx, dir, "ls-files", "--stage", "-z") + if err != nil { + return "", err + } type entry struct { path string mode string digest [32]byte } - fsys := os.DirFS(root) var entries []entry - err := fs.WalkDir(fsys, ".", func(path string, dirEntry fs.DirEntry, err error) error { - if err != nil { - return err - } - if dirEntry.IsDir() { - if dirEntry.Name() == gitDir { - return fs.SkipDir - } - return nil - } - if !dirEntry.Type().IsRegular() { - return nil - } - data, err := fs.ReadFile(fsys, path) - if err != nil { - return err + // `ls-files --stage -z` records are " \t\0". + for _, record := range strings.Split(strings.TrimRight(out, "\x00"), "\x00") { + if record == "" { + continue } - info, err := dirEntry.Info() - if err != nil { - return err + meta, path, ok := strings.Cut(record, "\t") + if !ok { + continue } + fields := strings.Fields(meta) mode := "644" - if info.Mode()&0o111 != 0 { + if len(fields) > 0 && strings.HasSuffix(fields[0], "755") { mode = "755" } + data, err := os.ReadFile(filepath.Join(dir, filepath.FromSlash(path))) + if err != nil { + return "", err + } entries = append(entries, entry{path: path, mode: mode, digest: sha256.Sum256(data)}) - return nil - }) - if err != nil { - return "", err } sort.Slice(entries, func(i, j int) bool { return entries[i].path < entries[j].path }) hasher := sha256.New() diff --git a/pkg/librarystore/github_test.go b/pkg/librarystore/github_test.go index 239c5a1c..8a4cb02b 100644 --- a/pkg/librarystore/github_test.go +++ b/pkg/librarystore/github_test.go @@ -133,6 +133,25 @@ func TestGitHubStoreRejectsModulePathMismatchBeforeTouchingTheRemote(t *testing. require.Contains(t, err.Error(), "go.mod") } +func TestGitHubStorePublishIgnoresAmbientSigningConfig(t *testing.T) { + ctx := context.Background() + // A global git config that forces signing with a bogus program would make an + // unguarded commit or tag fail; Publish must override it per-invocation. + globalConfig := filepath.Join(t.TempDir(), "gitconfig") + require.NoError(t, os.WriteFile(globalConfig, + []byte("[commit]\n\tgpgsign = true\n[tag]\n\tgpgsign = true\n[gpg]\n\tprogram = /bin/false\n"), 0o644)) + t.Setenv("GIT_CONFIG_GLOBAL", globalConfig) + t.Setenv("GIT_CONFIG_SYSTEM", os.DevNull) + + remote := bareRepo(t) + store := storeTo(remote) + modulePath := goModulePath(remote) + + _, err := store.Publish(ctx, goModule(t, modulePath, "package authkit\n"), + Coordinates{Language: LanguageGo, Name: "authkit", Version: "1.0.0"}) + require.NoError(t, err) +} + func TestGitHubStoreRejectsNonSemverAndUnsupportedLanguages(t *testing.T) { ctx := context.Background() store := storeTo(bareRepo(t)) @@ -208,6 +227,11 @@ func TestGoModulePathAndRepositoryName(t *testing.T) { func TestDefaultRemoteFor(t *testing.T) { store := NewGitHubStore("codefly-dev") require.Equal(t, "https://github.com/codefly-dev/authkit-go.git", store.remoteFor(LanguageGo, "authkit")) + + // Owner is read live, so reassigning it after construction retargets the + // store instead of silently publishing to the original owner. + store.Owner = "acme" + require.Equal(t, "https://github.com/acme/authkit-go.git", store.remoteFor(LanguageGo, "authkit")) } func TestParseGoModulePath(t *testing.T) { @@ -223,37 +247,69 @@ func TestParseGoModulePath(t *testing.T) { require.Error(t, err) } -func TestTreeDigestDeterministicContentAndModeSensitiveGitExcluded(t *testing.T) { - a := goModule(t, "example.com/x", "package x\n") - b := goModule(t, "example.com/x", "package x\n") - c := goModule(t, "example.com/x", "package x // different\n") +func TestTreeDigestCoversTrackedContentOnlyModeSensitive(t *testing.T) { + ctx := context.Background() + store := NewGitHubStore("codefly-dev") - da, err := treeDigest(a) - require.NoError(t, err) - db, err := treeDigest(b) + initRepo := func(t *testing.T) string { + t.Helper() + dir := t.TempDir() + for _, args := range [][]string{ + {"init", "--quiet"}, + {"config", "user.email", "t@t"}, + {"config", "user.name", "t"}, + {"config", "commit.gpgsign", "false"}, + } { + cmd := exec.Command("git", args...) + cmd.Dir = dir + require.NoError(t, cmd.Run(), strings.Join(args, " ")) + } + return dir + } + commit := func(t *testing.T, dir string) { + t.Helper() + for _, args := range [][]string{{"add", "-A"}, {"commit", "--quiet", "-m", "x"}} { + cmd := exec.Command("git", args...) + cmd.Dir = dir + require.NoError(t, cmd.Run(), strings.Join(args, " ")) + } + } + + a := initRepo(t) + require.NoError(t, os.WriteFile(filepath.Join(a, "lib.go"), []byte("package x\n"), 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(a, ".gitignore"), []byte("secret.txt\n"), 0o644)) + commit(t, a) + da, err := store.treeDigest(ctx, a) require.NoError(t, err) - dc, err := treeDigest(c) + require.True(t, strings.HasPrefix(da, "sha256:")) + + // Byte-identical committed content yields an identical digest. + b := initRepo(t) + require.NoError(t, os.WriteFile(filepath.Join(b, "lib.go"), []byte("package x\n"), 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(b, ".gitignore"), []byte("secret.txt\n"), 0o644)) + commit(t, b) + db, err := store.treeDigest(ctx, b) require.NoError(t, err) require.Equal(t, da, db) - require.NotEqual(t, da, dc) - // Flipping the executable bit is a real change — git preserves it into the - // published tree. - require.NoError(t, os.Chmod(filepath.Join(b, "lib.go"), 0o755)) - dbExec, err := treeDigest(b) + // A .gitignore'd, untracked file is never published, so it must not change + // the digest — otherwise the publish tree and a clone would disagree. + require.NoError(t, os.WriteFile(filepath.Join(a, "secret.txt"), []byte("shhh\n"), 0o644)) + daIgnored, err := store.treeDigest(ctx, a) require.NoError(t, err) - require.NotEqual(t, da, dbExec) + require.Equal(t, da, daIgnored) - // A non-executable mode variation git cannot preserve does not change it. - require.NoError(t, os.Chmod(filepath.Join(a, "lib.go"), 0o640)) - daTightened, err := treeDigest(a) + // Different tracked content changes the digest. + require.NoError(t, os.WriteFile(filepath.Join(b, "lib.go"), []byte("package x // different\n"), 0o644)) + commit(t, b) + dbChanged, err := store.treeDigest(ctx, b) require.NoError(t, err) - require.Equal(t, da, daTightened) + require.NotEqual(t, db, dbChanged) - // .git content is never part of the published tree. - require.NoError(t, os.MkdirAll(filepath.Join(a, ".git"), 0o755)) - require.NoError(t, os.WriteFile(filepath.Join(a, ".git", "config"), []byte("[core]\n"), 0o644)) - daWithGit, err := treeDigest(a) + // Flipping the executable bit is a real change git carries into the tree. + require.NoError(t, os.Chmod(filepath.Join(a, "lib.go"), 0o755)) + commit(t, a) + daExec, err := store.treeDigest(ctx, a) require.NoError(t, err) - require.Equal(t, da, daWithGit) + require.NotEqual(t, da, daExec) } From 590877ddd1dad71e4e9691546416dbc25e560927 Mon Sep 17 00:00:00 2001 From: Antoine Toussaint Date: Sat, 22 Aug 2026 11:47:07 -0400 Subject: [PATCH 4/4] fix(librarystore): validate coordinates, digest blob IDs, keep pushes last MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Second-pass review found three ways a publish could go wrong before or after the irreversible push, all confirmed empirically: - Path traversal: owner/name were interpolated into the remote URL unvalidated, and git's HTTP client normalizes "owner/../../evil/repo" to a different repository — a crafted library name could redirect a credentialed publish. Both atoms are now validated as plain tokens at every entry point, before any git command runs. - Digest instability: treeDigest hashed working-tree bytes, so ambient core.autocrlf (deliberately preserved for credentials) made the publish-side digest differ from a resolve-side clone of the same release — verification would flag legitimate content as tampered. The digest is now over the index's blob object IDs, which are content-addressed after git's own normalization and provably identical on both sides (and require no file reads). - Unfetchable releases: build-metadata semver ("1.0.0+build.1") parses, tags, and pushes fine, but Go module versions discard build metadata, so the immutable tag could never be `go get`-ed. Publish now refuses it. Also: rev-parse/digest moved before the pushes so every failure leaves the remote untouched and retryable; the verbatim tag name is carried through resolution instead of being reconstructed from normalized semver (a human-added "v1.0" no longer breaks Resolve); output() folds git's stderr into errors instead of an opaque "exit status 128"; commit runs --no-verify so ambient hooks cannot fail or mutate a release. Co-Authored-By: Claude Fable 5 --- pkg/librarystore/github.go | 120 ++++++++++++++++++++++++-------- pkg/librarystore/github_test.go | 107 +++++++++++++++++++++++++--- 2 files changed, 186 insertions(+), 41 deletions(-) diff --git a/pkg/librarystore/github.go b/pkg/librarystore/github.go index b80636a8..3fa8c1fc 100644 --- a/pkg/librarystore/github.go +++ b/pkg/librarystore/github.go @@ -4,6 +4,7 @@ import ( "context" "crypto/sha256" "encoding/hex" + "errors" "fmt" "io/fs" "os" @@ -48,6 +49,37 @@ func repositoryName(language Language, name string) string { return fmt.Sprintf("%s-%s", name, language) } +// validCoordinateToken reports whether value can be embedded in a repository +// URL and passed on a git command line: a plain alphanumeric/dot/dash/underscore +// token. A ".." would let a crafted name traverse the URL path — git's HTTP +// client normalizes "owner/../../evil/repo" to a different repository — and a +// leading "-" could read as an option. +func validCoordinateToken(value string) bool { + if value == "" || len(value) > 100 || strings.HasPrefix(value, ".") || strings.HasPrefix(value, "-") || strings.Contains(value, "..") { + return false + } + for _, r := range value { + if (r < 'a' || r > 'z') && (r < 'A' || r > 'Z') && (r < '0' || r > '9') && r != '-' && r != '_' && r != '.' { + return false + } + } + return true +} + +// validateTarget guards the owner and library name before they reach a remote +// URL. Library names arrive from workspace configuration, which is a trust +// boundary: an unvalidated name could redirect a credentialed publish to a +// repository the coordinates never named. +func (s *GitHubStore) validateTarget(name string) error { + if !validCoordinateToken(s.Owner) { + return fmt.Errorf("librarystore: invalid owner %q", s.Owner) + } + if !validCoordinateToken(name) { + return fmt.Errorf("librarystore: invalid library name %q", name) + } + return nil +} + // goModulePath derives the Go module path from a remote URL: the identity a // consumer passes to `go get`. It is only meaningful for real remote URLs; a // local test remote yields a path used purely for assertions. @@ -67,10 +99,19 @@ func (s *GitHubStore) Publish(ctx context.Context, artifactDir string, c Coordin if c.Language != LanguageGo { return Published{}, fmt.Errorf("librarystore: publishing %s libraries is not implemented yet", c.Language) } + if err := s.validateTarget(c.Name); err != nil { + return Published{}, err + } version, err := semver.NewVersion(strings.TrimPrefix(c.Version, "v")) if err != nil { return Published{}, fmt.Errorf("librarystore: %q is not a semantic version: %w", c.Version, err) } + // Go module versions are canonical semver: build metadata is discarded by + // the go command, so a tag carrying it could never be fetched — and once + // pushed it would be immutable. Refuse it before anything irreversible. + if version.Metadata() != "" { + return Published{}, fmt.Errorf("librarystore: %q carries build metadata, which Go module versions do not support", c.Version) + } c.Version = version.String() remote := s.remoteFor(c.Language, c.Name) importPath := goModulePath(remote) @@ -109,10 +150,12 @@ func (s *GitHubStore) Publish(ctx context.Context, artifactDir string, c Coordin message := fmt.Sprintf("release %s %s", c.Name, tag) // --allow-empty: a release whose content is identical to the previous one // (a version bump with no code change) is a valid release, not an error. + // --no-verify: ambient hooks (core.hooksPath, clone templates) belong to + // the user's own projects and must not fail or mutate a release commit. if err = s.git(ctx, work, "-c", "user.name="+s.commitName, "-c", "user.email="+s.commitEmail, "-c", "commit.gpgsign=false", - "commit", "--quiet", "--allow-empty", "-m", message); err != nil { + "commit", "--quiet", "--allow-empty", "--no-verify", "-m", message); err != nil { return Published{}, fmt.Errorf("commit release: %w", err) } if err = s.git(ctx, work, @@ -121,12 +164,9 @@ func (s *GitHubStore) Publish(ctx context.Context, artifactDir string, c Coordin "tag", "-a", tag, "-m", message); err != nil { return Published{}, fmt.Errorf("tag release: %w", err) } - if err = s.git(ctx, work, "push", "--quiet", "origin", branch); err != nil { - return Published{}, fmt.Errorf("push branch: %w", err) - } - if err = s.git(ctx, work, "push", "--quiet", "origin", tag); err != nil { - return Published{}, fmt.Errorf("push tag: %w", err) - } + // Resolve the commit and digest before pushing: every failure up to here + // leaves the remote untouched and the publish cleanly retryable. A failure + // after the push would report an error for a release that is already live. commit, err := s.output(ctx, work, "rev-parse", tag+"^{commit}") if err != nil { return Published{}, err @@ -135,6 +175,12 @@ func (s *GitHubStore) Publish(ctx context.Context, artifactDir string, c Coordin if err != nil { return Published{}, err } + if err = s.git(ctx, work, "push", "--quiet", "origin", branch); err != nil { + return Published{}, fmt.Errorf("push branch: %w", err) + } + if err = s.git(ctx, work, "push", "--quiet", "origin", tag); err != nil { + return Published{}, fmt.Errorf("push tag: %w", err) + } return s.published(c, remote, strings.TrimSpace(commit), digest), nil } @@ -176,6 +222,9 @@ func (s *GitHubStore) Resolve(ctx context.Context, language Language, name, cons if language != LanguageGo { return Published{}, fmt.Errorf("librarystore: resolving %s libraries is not implemented yet", language) } + if err := s.validateTarget(name); err != nil { + return Published{}, err + } remote := s.remoteFor(language, name) tagged, err := s.listTagged(ctx, remote) if err != nil { @@ -199,9 +248,9 @@ func (s *GitHubStore) Resolve(ctx context.Context, language Language, name, cons return Published{}, fmt.Errorf("librarystore: no published version of %s satisfies %q", name, constraint) } if !isCommitHash(best.ref) { - return Published{}, fmt.Errorf("librarystore: tag %s of %s resolves to invalid commit %q", versionTag(best.version.String()), name, best.ref) + return Published{}, fmt.Errorf("librarystore: tag %s of %s resolves to invalid commit %q", best.tag, name, best.ref) } - digest, err := s.digestAtTag(ctx, remote, versionTag(best.version.String())) + digest, err := s.digestAtTag(ctx, remote, best.tag) if err != nil { return Published{}, err } @@ -213,6 +262,9 @@ func (s *GitHubStore) List(ctx context.Context, language Language, name string) if language != LanguageGo { return nil, fmt.Errorf("librarystore: listing %s libraries is not implemented yet", language) } + if err := s.validateTarget(name); err != nil { + return nil, err + } tagged, err := s.listTagged(ctx, s.remoteFor(language, name)) if err != nil { return nil, err @@ -225,8 +277,12 @@ func (s *GitHubStore) List(ctx context.Context, language Language, name string) } // taggedVersion is one published semver tag with the commit it resolves to. +// The tag name is carried through verbatim: reconstructing it from the parsed +// version would break on a non-canonical tag like "v1.0", whose normalized +// form names a ref that does not exist. type taggedVersion struct { version *semver.Version + tag string ref string } @@ -249,6 +305,7 @@ func (s *GitHubStore) listTagged(ctx context.Context, remote string) ([]taggedVe func parseTagListing(out string) []taggedVersion { type entry struct { version *semver.Version + tag string ref string peeled bool } @@ -273,11 +330,11 @@ func parseTagListing(out string) []taggedVersion { if existing, ok := byVersion[key]; ok && existing.peeled && !peeled { continue } - byVersion[key] = entry{version: v, ref: fields[0], peeled: peeled} + byVersion[key] = entry{version: v, tag: tag, ref: fields[0], peeled: peeled} } result := make([]taggedVersion, 0, len(byVersion)) for _, e := range byVersion { - result = append(result, taggedVersion{version: e.version, ref: e.ref}) + result = append(result, taggedVersion{version: e.version, tag: e.tag, ref: e.ref}) } sort.Slice(result, func(i, j int) bool { return result[i].version.GreaterThan(result[j].version) }) return result @@ -351,6 +408,10 @@ func (s *GitHubStore) output(ctx context.Context, dir string, args ...string) (s command.Env = gitEnv() out, err := command.Output() if err != nil { + var exitErr *exec.ExitError + if errors.As(err, &exitErr) && len(exitErr.Stderr) > 0 { + return "", fmt.Errorf("git %s: %w: %s", strings.Join(args, " "), err, strings.TrimSpace(string(exitErr.Stderr))) + } return "", fmt.Errorf("git %s: %w", strings.Join(args, " "), err) } return string(out), nil @@ -422,24 +483,24 @@ func replaceTrackedTree(work, source string) error { }) } -// treeDigest is a deterministic sha256 over the git-tracked files in dir, keyed -// by slash-separated relative path and git-normalized mode. Driving the file -// set from `git ls-files` — rather than a filesystem walk — means the digest -// describes exactly the published content: files a .gitignore excludes are -// never committed and never hashed, so the publish-side working tree and a -// resolve-side clone of the same version produce the same digest. Git preserves -// only the executable bit, so modes fold to 755/644 — capturing an -// executable-bit flip (a real change, per the docker-build-recipe v2 contract) -// while ignoring mode variations git itself does not carry. +// treeDigest is a deterministic sha256 over dir's git index: each tracked +// file's slash-separated path, git-normalized mode, and blob object ID. Blob +// IDs are content-addressed after git's own normalization (line endings, +// filters), so the publish-side tree and a resolve-side clone of the same +// release always agree; hashing working-tree bytes instead would make the +// digest vary with ambient core.autocrlf. Files a .gitignore excludes are +// never committed and never hashed. Git preserves only the executable bit, so +// modes fold to 755/644 — an executable-bit flip is a real change (per the +// docker-build-recipe v2 contract), other mode variations are not carried. func (s *GitHubStore) treeDigest(ctx context.Context, dir string) (string, error) { out, err := s.output(ctx, dir, "ls-files", "--stage", "-z") if err != nil { return "", err } type entry struct { - path string - mode string - digest [32]byte + path string + mode string + oid string } var entries []entry // `ls-files --stage -z` records are " \t\0". @@ -452,20 +513,19 @@ func (s *GitHubStore) treeDigest(ctx context.Context, dir string) (string, error continue } fields := strings.Fields(meta) + if len(fields) < 2 { + continue + } mode := "644" - if len(fields) > 0 && strings.HasSuffix(fields[0], "755") { + if strings.HasSuffix(fields[0], "755") { mode = "755" } - data, err := os.ReadFile(filepath.Join(dir, filepath.FromSlash(path))) - if err != nil { - return "", err - } - entries = append(entries, entry{path: path, mode: mode, digest: sha256.Sum256(data)}) + entries = append(entries, entry{path: path, mode: mode, oid: fields[1]}) } sort.Slice(entries, func(i, j int) bool { return entries[i].path < entries[j].path }) hasher := sha256.New() for _, e := range entries { - fmt.Fprintf(hasher, "%s\x00%s\x00%s\n", e.path, e.mode, hex.EncodeToString(e.digest[:])) + fmt.Fprintf(hasher, "%s\x00%s\x00%s\n", e.path, e.mode, e.oid) } return "sha256:" + hex.EncodeToString(hasher.Sum(nil)), nil } diff --git a/pkg/librarystore/github_test.go b/pkg/librarystore/github_test.go index 8a4cb02b..ae1f129f 100644 --- a/pkg/librarystore/github_test.go +++ b/pkg/librarystore/github_test.go @@ -133,13 +133,16 @@ func TestGitHubStoreRejectsModulePathMismatchBeforeTouchingTheRemote(t *testing. require.Contains(t, err.Error(), "go.mod") } -func TestGitHubStorePublishIgnoresAmbientSigningConfig(t *testing.T) { +func TestGitHubStorePublishIgnoresAmbientSigningAndHookConfig(t *testing.T) { ctx := context.Background() - // A global git config that forces signing with a bogus program would make an - // unguarded commit or tag fail; Publish must override it per-invocation. + // A global git config that forces signing with a bogus program, or points + // core.hooksPath at a hook that fails outside the user's own projects, + // would break an unguarded commit or tag; Publish must neutralize both. + hooks := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(hooks, "pre-commit"), []byte("#!/bin/sh\nexit 1\n"), 0o755)) globalConfig := filepath.Join(t.TempDir(), "gitconfig") require.NoError(t, os.WriteFile(globalConfig, - []byte("[commit]\n\tgpgsign = true\n[tag]\n\tgpgsign = true\n[gpg]\n\tprogram = /bin/false\n"), 0o644)) + []byte("[commit]\n\tgpgsign = true\n[tag]\n\tgpgsign = true\n[gpg]\n\tprogram = /bin/false\n[core]\n\thooksPath = "+hooks+"\n"), 0o644)) t.Setenv("GIT_CONFIG_GLOBAL", globalConfig) t.Setenv("GIT_CONFIG_SYSTEM", os.DevNull) @@ -159,10 +162,37 @@ func TestGitHubStoreRejectsNonSemverAndUnsupportedLanguages(t *testing.T) { _, err := store.Publish(ctx, t.TempDir(), Coordinates{Language: LanguageGo, Name: "authkit", Version: "latest"}) require.Error(t, err) + // Build metadata is valid semver but not a valid Go module version: the go + // command discards it, so the tag could never be fetched once pushed. + _, err = store.Publish(ctx, t.TempDir(), Coordinates{Language: LanguageGo, Name: "authkit", Version: "1.0.0+build.1"}) + require.ErrorContains(t, err, "build metadata") + _, err = store.Publish(ctx, t.TempDir(), Coordinates{Language: LanguagePython, Name: "authkit", Version: "1.0.0"}) require.ErrorContains(t, err, "not implemented") } +func TestGitHubStoreRejectsUnsafeCoordinates(t *testing.T) { + ctx := context.Background() + // The default remoteFor is kept: validation must fire before any remote is + // contacted, so a traversal name never reaches a git command at all. + store := NewGitHubStore("codefly-dev") + + // git's HTTP client normalizes "owner/../../evil/repo" to a different + // repository, so a crafted name would redirect a credentialed publish. + _, err := store.Publish(ctx, t.TempDir(), Coordinates{Language: LanguageGo, Name: "../../evil/lib", Version: "1.0.0"}) + require.ErrorContains(t, err, "invalid library name") + _, err = store.Resolve(ctx, LanguageGo, "../../evil/lib", "^1.0.0") + require.ErrorContains(t, err, "invalid library name") + _, err = store.List(ctx, LanguageGo, "--upload-pack=evil") + require.ErrorContains(t, err, "invalid library name") + _, err = store.List(ctx, LanguageGo, "a..b") + require.ErrorContains(t, err, "invalid library name") + + store.Owner = "codefly-dev/../evil" + _, err = store.List(ctx, LanguageGo, "authkit") + require.ErrorContains(t, err, "invalid owner") +} + func TestReplaceTrackedTreeSkipsSourceGitAndPreservesCloneGit(t *testing.T) { work := t.TempDir() require.NoError(t, os.MkdirAll(filepath.Join(work, ".git"), 0o755)) @@ -190,24 +220,79 @@ func TestReplaceTrackedTreeSkipsSourceGitAndPreservesCloneGit(t *testing.T) { require.Equal(t, "package lib\n", string(data)) } +func TestTreeDigestAgreesAcrossLineEndingConfig(t *testing.T) { + ctx := context.Background() + store := NewGitHubStore("codefly-dev") + + run := func(dir string, args ...string) { + t.Helper() + cmd := exec.Command("git", args...) + cmd.Dir = dir + out, err := cmd.CombinedOutput() + require.NoError(t, err, "git %v: %s", args, out) + } + + // Publish side: core.autocrlf=input stores an LF blob while the working + // tree keeps CRLF bytes. A digest over working-tree bytes would differ from + // a plain clone's LF checkout; a digest over blob IDs cannot. + pub := t.TempDir() + run(pub, "init", "--quiet") + run(pub, "config", "core.autocrlf", "input") + run(pub, "config", "user.email", "t@t") + run(pub, "config", "user.name", "t") + run(pub, "config", "commit.gpgsign", "false") + require.NoError(t, os.WriteFile(filepath.Join(pub, "lib.go"), []byte("package x\r\n"), 0o644)) + run(pub, "add", "-A") + run(pub, "commit", "--quiet", "-m", "x") + + clone := filepath.Join(t.TempDir(), "clone") + run(pub, "clone", "--quiet", "-c", "core.autocrlf=false", pub, clone) + + dPub, err := store.treeDigest(ctx, pub) + require.NoError(t, err) + dClone, err := store.treeDigest(ctx, clone) + require.NoError(t, err) + require.Equal(t, dPub, dClone) +} + +func TestOutputSurfacesGitStderr(t *testing.T) { + ctx := context.Background() + store := storeTo(filepath.Join(t.TempDir(), "missing.git")) + + // A failing ls-remote must carry git's own diagnosis, not an opaque + // "exit status 128" that hides an auth failure from a typo'd remote. + _, err := store.List(ctx, LanguageGo, "authkit") + require.Error(t, err) + require.Contains(t, err.Error(), "repository") +} + func TestParseTagListingPeeledWinsAndLightweightFallsBack(t *testing.T) { annotatedObject := strings.Repeat("a", 40) annotatedCommit := strings.Repeat("b", 40) lightweightCommit := strings.Repeat("c", 40) + nonCanonicalCommit := strings.Repeat("f", 40) out := annotatedObject + "\trefs/tags/v1.0.0\n" + annotatedCommit + "\trefs/tags/v1.0.0^{}\n" + lightweightCommit + "\trefs/tags/v2.0.0\n" + strings.Repeat("d", 40) + "\trefs/heads/main\n" + - strings.Repeat("e", 40) + "\trefs/tags/not-semver\n" + strings.Repeat("e", 40) + "\trefs/tags/not-semver\n" + + nonCanonicalCommit + "\trefs/tags/v3.0\n" tagged := parseTagListing(out) - require.Len(t, tagged, 2) - // Newest first; a lightweight tag's hash is already the commit. - require.Equal(t, "2.0.0", tagged[0].version.String()) - require.Equal(t, lightweightCommit, tagged[0].ref) + require.Len(t, tagged, 3) + // A non-canonical tag keeps its verbatim name: reconstructing "v3.0.0" from + // the parsed version would name a ref that does not exist. + require.Equal(t, "3.0.0", tagged[0].version.String()) + require.Equal(t, "v3.0", tagged[0].tag) + require.Equal(t, nonCanonicalCommit, tagged[0].ref) + // A lightweight tag's hash is already the commit. + require.Equal(t, "2.0.0", tagged[1].version.String()) + require.Equal(t, "v2.0.0", tagged[1].tag) + require.Equal(t, lightweightCommit, tagged[1].ref) // The peeled commit wins over the annotated tag object. - require.Equal(t, "1.0.0", tagged[1].version.String()) - require.Equal(t, annotatedCommit, tagged[1].ref) + require.Equal(t, "1.0.0", tagged[2].version.String()) + require.Equal(t, "v1.0.0", tagged[2].tag) + require.Equal(t, annotatedCommit, tagged[2].ref) } func TestIsCommitHash(t *testing.T) {