From f7bf413cd6831d5873bee2d0c30aecae5ee4badb Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Fri, 14 Aug 2026 17:06:56 +0200 Subject: [PATCH 1/5] test(e2e): declarative Scenario layer, image-identity tests as demonstrator Introduce a small declarative layer over the e2e framework so a test reads as its intent: an inline compose.yaml, a sequence of steps (command + expected observables), and nothing else. The framework owns project naming (derived from the test name), cleanup, and failure reporting: on the first unmet expectation the test fails with a step transcript, the failing command output, the project's container state, the engine events since the scenario started and the container logs. Checks observe real state (container IDs across steps, labels, image manifest platform) instead of grepping the progress writer output, which stays available as an explicit last-resort check. The e2e tests introduced by #14041 are converted as a demonstrator; their dedicated fixture files are inlined in the scenarios and removed. Signed-off-by: Nicolas De Loof --- .../image-identity/mixed-platforms.yaml | 8 - .../image-identity/refresh-window.yaml | 5 - pkg/e2e/fixtures/multiplatform/compose.yaml | 5 - pkg/e2e/image_identity_corner_test.go | 192 +++--- pkg/e2e/multiplatform_test.go | 68 +-- pkg/e2e/scenario.go | 564 ++++++++++++++++++ pkg/e2e/volumes_test.go | 45 +- 7 files changed, 688 insertions(+), 199 deletions(-) delete mode 100644 pkg/e2e/fixtures/image-identity/mixed-platforms.yaml delete mode 100644 pkg/e2e/fixtures/image-identity/refresh-window.yaml delete mode 100644 pkg/e2e/fixtures/multiplatform/compose.yaml create mode 100644 pkg/e2e/scenario.go diff --git a/pkg/e2e/fixtures/image-identity/mixed-platforms.yaml b/pkg/e2e/fixtures/image-identity/mixed-platforms.yaml deleted file mode 100644 index ec3f3fa6d5..0000000000 --- a/pkg/e2e/fixtures/image-identity/mixed-platforms.yaml +++ /dev/null @@ -1,8 +0,0 @@ -services: - native: - image: alpine:3.19 - command: ["sleep", "infinity"] - pinned: - image: alpine:3.19 - platform: ${PINNED_PLATFORM} - command: ["sleep", "infinity"] diff --git a/pkg/e2e/fixtures/image-identity/refresh-window.yaml b/pkg/e2e/fixtures/image-identity/refresh-window.yaml deleted file mode 100644 index 8e7311bc9b..0000000000 --- a/pkg/e2e/fixtures/image-identity/refresh-window.yaml +++ /dev/null @@ -1,5 +0,0 @@ -services: - app: - image: alpine:3.18 - pull_policy: daily - command: ["sleep", "infinity"] diff --git a/pkg/e2e/fixtures/multiplatform/compose.yaml b/pkg/e2e/fixtures/multiplatform/compose.yaml deleted file mode 100644 index d93820b76d..0000000000 --- a/pkg/e2e/fixtures/multiplatform/compose.yaml +++ /dev/null @@ -1,5 +0,0 @@ -services: - repro: - image: compose-e2e-multiplatform-local-only:v1 - platform: ${REQUESTED_PLATFORM} - command: ["uname", "-m"] diff --git a/pkg/e2e/image_identity_corner_test.go b/pkg/e2e/image_identity_corner_test.go index 7741a46f53..33be3a2460 100644 --- a/pkg/e2e/image_identity_corner_test.go +++ b/pkg/e2e/image_identity_corner_test.go @@ -17,41 +17,25 @@ package e2e import ( - "fmt" - "strings" "testing" - - "gotest.tools/v3/assert" - "gotest.tools/v3/icmd" ) -// nonNativePlatform returns a linux platform different from the daemon's. -func nonNativePlatform(t *testing.T, c *CLI) string { - t.Helper() - arch := c.RunDockerCmd(t, "info", "--format", "{{.Architecture}}").Stdout() - if strings.Contains(arch, "x86_64") { - return "linux/arm64" - } - return "linux/amd64" -} - // TestUpDryRunMissingImage: the dry-run client fakes the pull, so the // post-pull identity resolution must not inspect the real daemon for an image // that was never actually pulled — that failed with "No such image". func TestUpDryRunMissingImage(t *testing.T) { - c := NewParallelCLI(t) - const projectName = "compose-e2e-identity-dryrun" - const image = "alpine:3.20" - const composeFile = "./fixtures/image-identity/compose.yaml" - - t.Cleanup(func() { - c.RunDockerComposeCmdNoCheck(t, "--project-name", projectName, "down", "--timeout=0") - }) - c.RunDockerOrExitError(t, "rmi", "-f", image) - - res := c.RunDockerComposeCmdNoCheck(t, "--dry-run", "-f", composeFile, "--project-name", projectName, "up", "-d") - res.Assert(t, icmd.Success) - assert.Check(t, !strings.Contains(res.Combined(), "No such image"), res.Combined()) + NewScenario(t, "dry-run up with a locally-missing image must not resolve it against the real daemon"). + Compose(` +services: + app: + image: alpine:3.20 + command: ["sleep", "infinity"] +`). + Step("make sure the image is not in the local store", + DockerCmd("rmi", "-f", "alpine:3.20").MayFail()). + Step("dry-run up succeeds on the faked pull", + ComposeCmd("--dry-run", "up", "-d"), + OutputNotContains("No such image")) } // TestCreateIdempotentDefaultPlatform locks the invariant that with @@ -59,37 +43,27 @@ func TestUpDryRunMissingImage(t *testing.T) { // after the pull and the one recomputed from the local store on the next run // agree, whatever platform each resolution used. func TestCreateIdempotentDefaultPlatform(t *testing.T) { - c := NewCLI(t) - requireContainerdStore(t, c) - - const projectName = "compose-e2e-identity-default-platform" - const image = "alpine:3.20" - const composeFile = "./fixtures/image-identity/compose.yaml" - platform := nonNativePlatform(t, c) - - t.Cleanup(func() { - c.cleanupWithDown(t, projectName) - c.RunDockerOrExitError(t, "rmi", "-f", image) - }) - c.RunDockerOrExitError(t, "rmi", "-f", image) - - create := func() *icmd.Result { - // `create` exercises the same pull/label path as `up` without needing - // emulation to run the non-native binary - cmd := c.NewDockerComposeCmd(t, "-f", composeFile, "--project-name", projectName, "create") - cmd.Env = append(cmd.Env, "DOCKER_DEFAULT_PLATFORM="+platform) - res := icmd.RunCmd(cmd) - res.Assert(t, icmd.Success) - return res - } - - create() - containerID := c.RunDockerCmd(t, "inspect", fmt.Sprintf("%s-app-1", projectName), "-f", "{{.Id}}").Stdout() - - res := create() - assert.Check(t, !strings.Contains(res.Combined(), "Recreate"), "second `create` should not recreate anything, got: %s", res.Combined()) - newContainerID := c.RunDockerCmd(t, "inspect", fmt.Sprintf("%s-app-1", projectName), "-f", "{{.Id}}").Stdout() - assert.Equal(t, containerID, newContainerID) + s := NewScenario(t, "with DOCKER_DEFAULT_PLATFORM set to a non-native platform, an unchanged create must not recreate", Serial()). + Requires(ContainerdImageStore) + + // `create` exercises the same pull/label path as `up` without needing + // emulation to run the non-native binary + s.Env("DOCKER_DEFAULT_PLATFORM="+s.NonNativePlatform()). + Compose(` +services: + app: + image: alpine:3.20 + command: ["sleep", "infinity"] +`). + Defer(DockerCmd("rmi", "-f", "alpine:3.20")). + Step("start without the image in the local store", + DockerCmd("rmi", "-f", "alpine:3.20").MayFail()). + Step("create pulls the image and records its identity", + ComposeCmd("create")). + Step("an unchanged create is a no-op", + ComposeCmd("create"), + OutputNotContains("Recreate"), + NotRecreated("app")) } // TestCreateIdempotentSharedImageMixedPlatforms: two services share the same @@ -99,67 +73,53 @@ func TestCreateIdempotentDefaultPlatform(t *testing.T) { // host default platform), and each service must be labeled with its own // platform's manifest digest — idempotently across runs. func TestCreateIdempotentSharedImageMixedPlatforms(t *testing.T) { - c := NewCLI(t) - requireContainerdStore(t, c) - - const projectName = "compose-e2e-identity-mixed-platforms" - const image = "alpine:3.19" - const composeFile = "./fixtures/image-identity/mixed-platforms.yaml" - platform := nonNativePlatform(t, c) - - t.Cleanup(func() { - c.cleanupWithDown(t, projectName) - c.RunDockerOrExitError(t, "rmi", "-f", image) - }) - c.RunDockerOrExitError(t, "rmi", "-f", image) - - create := func() *icmd.Result { - cmd := c.NewDockerComposeCmd(t, "-f", composeFile, "--project-name", projectName, "create") - cmd.Env = append(cmd.Env, "PINNED_PLATFORM="+platform) - res := icmd.RunCmd(cmd) - res.Assert(t, icmd.Success) - return res - } - - create() - label := func(service string) string { - return strings.TrimSpace(c.RunDockerCmd(t, "inspect", - fmt.Sprintf("%s-%s-1", projectName, service), - "-f", `{{index .Config.Labels "com.docker.compose.image"}}`).Stdout()) - } - nativeDigest, pinnedDigest := label("native"), label("pinned") - assert.Check(t, nativeDigest != "", "native service must be labeled") - assert.Check(t, pinnedDigest != "", "pinned service must be labeled") - assert.Check(t, nativeDigest != pinnedDigest, - "the two services must be labeled with their own platform's manifest digest") - - res := create() - assert.Check(t, !strings.Contains(res.Combined(), "Recreate"), - "second `create` should not recreate anything, got: %s", res.Combined()) - assert.Equal(t, label("native"), nativeDigest) - assert.Equal(t, label("pinned"), pinnedDigest) + s := NewScenario(t, "two services sharing an image, one platform-pinned, must each keep their own platform's manifest digest", Serial()). + Requires(ContainerdImageStore) + + s.Env("PINNED_PLATFORM="+s.NonNativePlatform()). + Compose(` +services: + native: + image: alpine:3.19 + command: ["sleep", "infinity"] + pinned: + image: alpine:3.19 + platform: ${PINNED_PLATFORM} + command: ["sleep", "infinity"] +`). + Defer(DockerCmd("rmi", "-f", "alpine:3.19")). + Step("start without the image in the local store", + DockerCmd("rmi", "-f", "alpine:3.19").MayFail()). + Step("create labels each service with its own platform's manifest digest", + ComposeCmd("create"), + LabelSet("native", "com.docker.compose.image"), + LabelSet("pinned", "com.docker.compose.image"), + LabelsDistinct("com.docker.compose.image", "native", "pinned")). + Step("an unchanged create is a no-op and keeps both digests", + ComposeCmd("create"), + OutputNotContains("Recreate"), + NotRecreated("native", "pinned"), + LabelUnchanged("native", "com.docker.compose.image"), + LabelUnchanged("pinned", "com.docker.compose.image")) } // TestPullRefreshWindowExplicitPull: pull_policy daily/weekly/every_N gates // `up`, but an explicit `compose pull` is the user's way to force a refresh // ahead of the window, so it must pull even when the image is fresh. func TestPullRefreshWindowExplicitPull(t *testing.T) { - c := NewParallelCLI(t) - const projectName = "compose-e2e-identity-refresh-window" - const image = "alpine:3.18" - const composeFile = "./fixtures/image-identity/refresh-window.yaml" - - t.Cleanup(func() { - c.RunDockerComposeCmdNoCheck(t, "--project-name", projectName, "down", "--timeout=0") - c.RunDockerOrExitError(t, "rmi", "-f", image) - }) - - // make the image fresh: the window (daily) is not due - c.RunDockerComposeCmd(t, "-f", composeFile, "--project-name", projectName, "pull") - - res := c.RunDockerComposeCmd(t, "-f", composeFile, "--project-name", projectName, "pull") - assert.Check(t, strings.Contains(res.Combined(), "Pulled"), - "explicit pull must refresh ahead of the window, got: %s", res.Combined()) - assert.Check(t, !strings.Contains(res.Combined(), "Skipped"), - "explicit pull must not skip on the refresh window, got: %s", res.Combined()) + NewScenario(t, "an explicit pull must refresh the image even when the pull_policy window is not due"). + Compose(` +services: + app: + image: alpine:3.18 + pull_policy: daily + command: ["sleep", "infinity"] +`). + Defer(DockerCmd("rmi", "-f", "alpine:3.18")). + Step("make the image fresh: the daily window is not due", + ComposeCmd("pull")). + Step("an explicit pull refreshes ahead of the window", + ComposeCmd("pull"), + OutputContains("Pulled"), + OutputNotContains("Skipped")) } diff --git a/pkg/e2e/multiplatform_test.go b/pkg/e2e/multiplatform_test.go index 49061f7668..fa7edcef93 100644 --- a/pkg/e2e/multiplatform_test.go +++ b/pkg/e2e/multiplatform_test.go @@ -17,11 +17,7 @@ package e2e import ( - "strings" "testing" - - "gotest.tools/v3/assert" - "gotest.tools/v3/icmd" ) func TestCreateLocalMultiPlatformImage(t *testing.T) { @@ -29,49 +25,35 @@ func TestCreateLocalMultiPlatformImage(t *testing.T) { // With the containerd image store, a local multi-platform image holding the // requested non-native variant satisfies the default "missing" pull policy: // compose must use it and not try to pull the (unpublished) tag. - c := NewParallelCLI(t) - - driverStatus := c.RunDockerCmd(t, "info", "--format", "{{json .DriverStatus}}").Stdout() - if !strings.Contains(driverStatus, "io.containerd.snapshotter.v1") { - t.Skip("containerd image store not enabled, can't hold a multi-platform image locally") - } + s := NewScenario(t, "a local multi-platform image must satisfy the default missing pull policy for a non-native platform"). + Requires(ContainerdImageStore) // request the non-native platform, so the requested variant can't be the - // one a platform-less image inspect reports - requested := "linux/amd64" - if arch := c.RunDockerCmd(t, "info", "--format", "{{.Architecture}}").Stdout(); strings.Contains(arch, "x86_64") { - requested = "linux/arm64" - } - - // the tag deliberately doesn't exist on any registry: resolving the - // requested platform from the local image is the only way to succeed + // one a platform-less image inspect reports; the tag deliberately doesn't + // exist on any registry: resolving the requested platform from the local + // image is the only way to succeed + requested := s.NonNativePlatform() const image = "compose-e2e-multiplatform-local-only:v1" - const projectName = "e2e-multiplatform-local" - - cleanup := func() { - c.RunDockerComposeCmdNoCheck(t, "--project-name", projectName, "down", "--timeout=0") - c.RunDockerOrExitError(t, "image", "rm", image) - } - cleanup() - t.Cleanup(cleanup) - - // store both the native and the requested variants under the local-only tag - c.RunDockerCmd(t, "pull", "-q", "--platform", "linux/amd64", "alpine:3.22") - c.RunDockerCmd(t, "pull", "-q", "--platform", "linux/arm64", "alpine:3.22") - c.RunDockerCmd(t, "tag", "alpine:3.22", image) // `create` exercises the same image-resolution/pull-policy path as `up`, // without requiring emulation to actually run the non-native binary - cmd := c.NewDockerComposeCmd(t, "-f", "./fixtures/multiplatform/compose.yaml", - "--project-name", projectName, "create") - cmd.Env = append(cmd.Env, "REQUESTED_PLATFORM="+requested) - res := icmd.RunCmd(cmd) - res.Assert(t, icmd.Success) - assert.Assert(t, !strings.Contains(res.Combined(), "Pulling"), res.Combined()) - - // the created container must be for the requested variant - platform := strings.TrimSpace(c.RunDockerCmd(t, "inspect", "--format", - "{{.ImageManifestDescriptor.Platform.OS}}/{{.ImageManifestDescriptor.Platform.Architecture}}", - projectName+"-repro-1").Stdout()) - assert.Equal(t, requested, platform) + s.Env("REQUESTED_PLATFORM="+requested). + Compose(` +services: + repro: + image: compose-e2e-multiplatform-local-only:v1 + platform: ${REQUESTED_PLATFORM} + command: ["uname", "-m"] +`). + Defer(DockerCmd("image", "rm", image)). + Step("store the amd64 variant in the local store", + DockerCmd("pull", "-q", "--platform", "linux/amd64", "alpine:3.22")). + Step("store the arm64 variant in the local store", + DockerCmd("pull", "-q", "--platform", "linux/arm64", "alpine:3.22")). + Step("tag both variants under a tag that exists on no registry", + DockerCmd("tag", "alpine:3.22", image)). + Step("create uses the local variant for the requested platform without pulling", + ComposeCmd("create"), + OutputNotContains("Pulling"), + RunsOnPlatform("repro", requested)) } diff --git a/pkg/e2e/scenario.go b/pkg/e2e/scenario.go new file mode 100644 index 0000000000..46b1a28a50 --- /dev/null +++ b/pkg/e2e/scenario.go @@ -0,0 +1,564 @@ +/* + Copyright 2026 Docker Compose CLI authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package e2e + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "slices" + "strconv" + "strings" + "testing" + "time" + "unicode" + + "gotest.tools/v3/icmd" +) + +// Scenario is a thin declarative layer over the e2e CLI helpers: a test reads +// as a compose.yaml, a sequence of steps (command + expected observables) and +// nothing else. Project naming, cleanup and failure diagnostics (transcript, +// project state, engine events, container logs) are handled by the framework. +// +// Steps execute eagerly: each Step call runs its command, snapshots the +// project containers and evaluates the checks, failing the test with a full +// scenario report on the first unmet expectation. +type Scenario struct { + t *testing.T + cli *CLI + intent string + project string + file string + env []string + parallel bool + start time.Time + deferred []Action + steps []stepRecord + snaps []snapshot +} + +type stepRecord struct { + name string + command string + result *icmd.Result + duration time.Duration +} + +// containerState is the per-container state captured after each step, used by +// cross-step checks such as NotRecreated or LabelUnchanged. +type containerState struct { + ID string + Name string + Labels map[string]string +} + +// snapshot maps a service name to its containers, sorted by name. +type snapshot map[string][]containerState + +// ScenarioOption customizes a Scenario at creation time. +type ScenarioOption func(*Scenario) + +// Serial disables the default parallel execution, for scenarios that mutate +// shared daemon state (e.g. removing images other tests may pull). +func Serial() ScenarioOption { + return func(s *Scenario) { s.parallel = false } +} + +// NewScenario creates a scenario named after the test, with a unique project +// name, an isolated CLI instance and automatic `down` cleanup registered. +// The intent is a one-line statement of the behavior being locked, displayed +// in logs and failure reports. +func NewScenario(t *testing.T, intent string, opts ...ScenarioOption) *Scenario { + t.Helper() + s := &Scenario{ + t: t, + intent: intent, + parallel: true, + project: projectNameFor(t.Name()), + start: time.Now(), + snaps: []snapshot{{}}, + } + for _, opt := range opts { + opt(s) + } + if s.parallel { + s.cli = NewParallelCLI(t) + } else { + s.cli = NewCLI(t) + } + t.Logf("scenario: %s (project %s)", intent, s.project) + + // start from — and return to — a clean slate, whatever previous runs left + s.cli.RunDockerComposeCmdNoCheck(t, "--project-name", s.project, "down", "-v", "--remove-orphans", "--timeout", "0") + t.Cleanup(func() { + s.cli.RunDockerComposeCmdNoCheck(t, "--project-name", s.project, "down", "-v", "--remove-orphans", "--timeout", "0") + for _, action := range s.deferred { + _ = icmd.RunCmd(s.command(action)) + } + }) + return s +} + +// projectNameFor derives a valid, readable compose project name from a test +// name: TestUpDryRunMissingImage -> e2e-up-dry-run-missing-image. +func projectNameFor(testName string) string { + name := strings.TrimPrefix(testName, "Test") + var b strings.Builder + var prev rune + for i, r := range name { + switch { + case unicode.IsUpper(r): + if i > 0 && !unicode.IsUpper(prev) { + b.WriteRune('-') + } + b.WriteRune(unicode.ToLower(r)) + case unicode.IsLower(r) || unicode.IsDigit(r): + b.WriteRune(r) + default: + b.WriteRune('-') + } + prev = r + } + return "e2e-" + strings.Trim(b.String(), "-") +} + +// CLI exposes the underlying CLI instance for the rare setup logic the +// declarative layer doesn't cover. +func (s *Scenario) CLI() *CLI { return s.cli } + +// Compose declares the project's compose model, written to a temporary +// directory so the whole scenario is self-contained in the test source. +func (s *Scenario) Compose(yaml string) *Scenario { + s.t.Helper() + dir := s.t.TempDir() + s.file = filepath.Join(dir, "compose.yaml") + if err := os.WriteFile(s.file, []byte(yaml), 0o644); err != nil { + s.t.Fatalf("failed to write compose.yaml: %v", err) + } + return s +} + +// Env sets environment variables applied to every subsequent step command +// (and interpolated in the compose model). +func (s *Scenario) Env(kv ...string) *Scenario { + s.env = append(s.env, kv...) + return s +} + +// Requires skips the scenario unless every requirement is met by the target +// environment. +func (s *Scenario) Requires(reqs ...Requirement) *Scenario { + s.t.Helper() + for _, req := range reqs { + if reason := req(s.t, s.cli); reason != "" { + s.t.Skip(reason) + } + } + return s +} + +// Defer registers a best-effort cleanup action executed after the project is +// taken down, e.g. removing images the scenario pulled. +func (s *Scenario) Defer(actions ...Action) *Scenario { + s.deferred = append(s.deferred, actions...) + return s +} + +// NonNativePlatform returns a linux platform different from the daemon's, +// for scenarios exercising platform-pinned services without emulation. +func (s *Scenario) NonNativePlatform() string { + s.t.Helper() + arch := s.cli.RunDockerCmd(s.t, "info", "--format", "{{.Architecture}}").Stdout() + if strings.Contains(arch, "x86_64") { + return "linux/arm64" + } + return "linux/amd64" +} + +// Step runs an action and asserts the expected observables. The command must +// succeed unless the action is marked MayFail. On the first unmet +// expectation the scenario fails with a transcript and project diagnostics. +func (s *Scenario) Step(name string, action Action, checks ...Check) *Scenario { + t := s.t + t.Helper() + cmd := s.command(action) + t.Logf("step: %s — %s", name, strings.Join(cmd.Command, " ")) + + begin := time.Now() + res := icmd.RunCmd(cmd) + rec := stepRecord{name: name, command: strings.Join(cmd.Command, " "), result: res, duration: time.Since(begin)} + s.steps = append(s.steps, rec) + + if !action.mayFail && res.ExitCode != 0 { + s.fail(fmt.Errorf("command exited with code %d", res.ExitCode)) + } + + s.snaps = append(s.snaps, s.snapshot()) + ctx := &CheckContext{ + scenario: s, + result: res, + prev: s.snaps[len(s.snaps)-2], + curr: s.snaps[len(s.snaps)-1], + } + for _, check := range checks { + if err := check.fn(ctx); err != nil { + s.fail(fmt.Errorf("expected %s: %w", check.name, err)) + } + } + return s +} + +// command materializes an action into a runnable command, layering scenario +// env then action env on top of the CLI environment. +func (s *Scenario) command(action Action) icmd.Cmd { + s.t.Helper() + var cmd icmd.Cmd + switch action.kind { + case kindCompose: + args := []string{} + if s.file != "" { + args = append(args, "-f", s.file) + } + args = append(args, "--project-name", s.project) + args = append(args, action.args...) + cmd = s.cli.NewDockerComposeCmd(s.t, args...) + case kindDocker: + cmd = s.cli.NewDockerCmd(s.t, action.args...) + } + cmd.Env = append(cmd.Env, s.env...) + cmd.Env = append(cmd.Env, action.env...) + return cmd +} + +// snapshot captures the current state of the project's containers. +func (s *Scenario) snapshot() snapshot { + s.t.Helper() + snap := snapshot{} + ids := s.projectContainerIDs() + if len(ids) == 0 { + return snap + } + res := icmd.RunCmd(s.cli.NewDockerCmd(s.t, append([]string{"inspect"}, ids...)...)) + if res.ExitCode != 0 { + return snap + } + var containers []struct { + ID string `json:"Id"` + Name string `json:"Name"` + Config struct { + Labels map[string]string `json:"Labels"` + } `json:"Config"` + } + if err := json.Unmarshal([]byte(res.Stdout()), &containers); err != nil { + return snap + } + for _, c := range containers { + service := c.Config.Labels["com.docker.compose.service"] + snap[service] = append(snap[service], containerState{ + ID: c.ID, + Name: strings.TrimPrefix(c.Name, "/"), + Labels: c.Config.Labels, + }) + } + for service := range snap { + slices.SortFunc(snap[service], func(a, b containerState) int { return strings.Compare(a.Name, b.Name) }) + } + return snap +} + +func (s *Scenario) projectContainerIDs() []string { + res := icmd.RunCmd(s.cli.NewDockerCmd(s.t, "ps", "-a", "--no-trunc", + "--filter", "label=com.docker.compose.project="+s.project, "--format", "{{.ID}}")) + if res.ExitCode != 0 || strings.TrimSpace(res.Stdout()) == "" { + return nil + } + return Lines(res.Stdout()) +} + +// fail reports the scenario failure: intent, step transcript, output of the +// failing command, then live diagnostics (project state, engine events since +// the scenario started, container logs). +func (s *Scenario) fail(reason error) { + t := s.t + t.Helper() + var b strings.Builder + fmt.Fprintf(&b, "scenario failed: %s\n", s.intent) + fmt.Fprintf(&b, "project: %s\n\ntranscript:\n", s.project) + for i, step := range s.steps { + mark := "✓" + if i == len(s.steps)-1 { + mark = "✗" + } + fmt.Fprintf(&b, " %s %s — %s (exit %d, %s)\n", mark, step.name, step.command, step.result.ExitCode, step.duration.Round(time.Millisecond)) + } + last := s.steps[len(s.steps)-1] + fmt.Fprintf(&b, "\nfailure: %v\n\n--- output of failing step\n%s\n", reason, truncate(last.result.Combined(), 4000)) + + fmt.Fprintf(&b, "\n--- project containers\n%s\n", truncate(s.diag("ps", "-a", "--no-trunc", "--filter", "label=com.docker.compose.project="+s.project), 2000)) + fmt.Fprintf(&b, "\n--- engine events since scenario start\n%s\n", truncate(s.diag("events", + "--since", s.start.Format(time.RFC3339Nano), "--until", time.Now().Format(time.RFC3339Nano), + "--filter", "label=com.docker.compose.project="+s.project), 2000)) + for _, containers := range s.snapshot() { + for _, c := range containers { + fmt.Fprintf(&b, "\n--- logs %s\n%s\n", c.Name, truncate(s.diag("logs", "--tail", "30", c.ID), 2000)) + } + } + t.Fatal(b.String()) +} + +func (s *Scenario) diag(args ...string) string { + res := icmd.RunCmd(s.cli.NewDockerCmd(s.t, args...)) + return strings.TrimSpace(res.Combined()) +} + +func truncate(out string, limit int) string { + if len(out) <= limit { + return out + } + return out[:limit] + fmt.Sprintf("\n… (%d more bytes)", len(out)-limit) +} + +// ---------- Actions ---------- + +type actionKind int + +const ( + kindCompose actionKind = iota + kindDocker +) + +// Action is a command a step executes: a compose command run against the +// scenario's project, or a raw docker command. +type Action struct { + kind actionKind + args []string + env []string + mayFail bool +} + +// ComposeCmd runs `docker compose ` against the scenario's compose +// file and project name. +func ComposeCmd(args ...string) Action { + return Action{kind: kindCompose, args: args} +} + +// DockerCmd runs a raw `docker ` command. +func DockerCmd(args ...string) Action { + return Action{kind: kindDocker, args: args} +} + +// WithEnv adds environment variables to this action only. +func (a Action) WithEnv(kv ...string) Action { + a.env = append(slices.Clone(a.env), kv...) + return a +} + +// MayFail marks the action as best-effort: a non-zero exit does not fail the +// scenario (e.g. removing an image that may not exist). +func (a Action) MayFail() Action { + a.mayFail = true + return a +} + +// ---------- Requirements ---------- + +// Requirement checks an environment prerequisite; it returns a non-empty +// skip reason when the requirement is not met. +type Requirement func(t testing.TB, c *CLI) string + +// ContainerdImageStore requires the daemon to use the containerd image store. +func ContainerdImageStore(t testing.TB, c *CLI) string { + t.Helper() + res := c.RunDockerCmd(t, "info", "-f", "{{json .DriverStatus}}") + if !strings.Contains(res.Stdout(), "io.containerd.snapshotter.v1") { + return "daemon is not using the containerd image store" + } + return "" +} + +// EngineVersionAtLeast requires a minimum daemon major version. +func EngineVersionAtLeast(major int) Requirement { + return func(t testing.TB, c *CLI) string { + t.Helper() + version := c.RunDockerCmd(t, "version", "-f", "{{.Server.Version}}").Combined() + before, _, _ := strings.Cut(strings.TrimSpace(version), ".") + if v, err := strconv.Atoi(before); err == nil && v < major { + return fmt.Sprintf("engine version %s < %d", strings.TrimSpace(version), major) + } + return "" + } +} + +// ---------- Checks ---------- + +// CheckContext gives a check access to the step's result and to the project +// state captured before and after the step. +type CheckContext struct { + scenario *Scenario + result *icmd.Result + prev snapshot + curr snapshot +} + +// Check is a named observable expected to hold after a step. +type Check struct { + name string + fn func(*CheckContext) error +} + +// OutputContains expects the command output to contain a string. Prefer +// state-based checks; use this when the CLI's reported decision is itself +// the observable. +func OutputContains(sub string) Check { + return Check{ + name: fmt.Sprintf("output contains %q", sub), + fn: func(ctx *CheckContext) error { + if !strings.Contains(ctx.result.Combined(), sub) { + return fmt.Errorf("not found in output") + } + return nil + }, + } +} + +// OutputNotContains expects the command output not to contain a string. +func OutputNotContains(sub string) Check { + return Check{ + name: fmt.Sprintf("output does not contain %q", sub), + fn: func(ctx *CheckContext) error { + if strings.Contains(ctx.result.Combined(), sub) { + return fmt.Errorf("found in output") + } + return nil + }, + } +} + +// NotRecreated expects the services' containers to be exactly the ones that +// existed before the step (same container IDs). +func NotRecreated(services ...string) Check { + return Check{ + name: fmt.Sprintf("services %s not recreated", strings.Join(services, ", ")), + fn: func(ctx *CheckContext) error { + for _, service := range services { + before, after := containerIDs(ctx.prev[service]), containerIDs(ctx.curr[service]) + if len(before) == 0 { + return fmt.Errorf("service %q had no container before the step", service) + } + if !slices.Equal(before, after) { + return fmt.Errorf("service %q containers changed: %v -> %v", service, before, after) + } + } + return nil + }, + } +} + +func containerIDs(containers []containerState) []string { + ids := make([]string, 0, len(containers)) + for _, c := range containers { + ids = append(ids, c.ID) + } + slices.Sort(ids) + return ids +} + +// LabelSet expects every container of the service to carry a non-empty label. +func LabelSet(service, key string) Check { + return Check{ + name: fmt.Sprintf("service %q has label %q set", service, key), + fn: func(ctx *CheckContext) error { + containers := ctx.curr[service] + if len(containers) == 0 { + return fmt.Errorf("service has no container") + } + for _, c := range containers { + if c.Labels[key] == "" { + return fmt.Errorf("label empty on container %s", c.Name) + } + } + return nil + }, + } +} + +// LabelsDistinct expects the given services to carry pairwise-distinct values +// for a label. +func LabelsDistinct(key string, services ...string) Check { + return Check{ + name: fmt.Sprintf("services %s have distinct %q labels", strings.Join(services, ", "), key), + fn: func(ctx *CheckContext) error { + seen := map[string]string{} + for _, service := range services { + containers := ctx.curr[service] + if len(containers) == 0 { + return fmt.Errorf("service %q has no container", service) + } + value := containers[0].Labels[key] + if other, dup := seen[value]; dup { + return fmt.Errorf("services %q and %q share label value %q", other, service, value) + } + seen[value] = service + } + return nil + }, + } +} + +// LabelUnchanged expects a service's label to have the same value as before +// the step. +func LabelUnchanged(service, key string) Check { + return Check{ + name: fmt.Sprintf("service %q label %q unchanged", service, key), + fn: func(ctx *CheckContext) error { + before, after := ctx.prev[service], ctx.curr[service] + if len(before) == 0 || len(after) == 0 { + return fmt.Errorf("service has no container to compare") + } + if before[0].Labels[key] != after[0].Labels[key] { + return fmt.Errorf("label changed: %q -> %q", before[0].Labels[key], after[0].Labels[key]) + } + return nil + }, + } +} + +// RunsOnPlatform expects the service's container to have been created for +// the given platform (from its image manifest descriptor). +func RunsOnPlatform(service, platform string) Check { + return Check{ + name: fmt.Sprintf("service %q container created for platform %s", service, platform), + fn: func(ctx *CheckContext) error { + containers := ctx.curr[service] + if len(containers) == 0 { + return fmt.Errorf("service has no container") + } + res := icmd.RunCmd(ctx.scenario.cli.NewDockerCmd(ctx.scenario.t, "inspect", "--format", + "{{.ImageManifestDescriptor.Platform.OS}}/{{.ImageManifestDescriptor.Platform.Architecture}}", + containers[0].ID)) + if res.ExitCode != 0 { + return fmt.Errorf("inspect failed: %s", res.Combined()) + } + if actual := strings.TrimSpace(res.Stdout()); actual != platform { + return fmt.Errorf("container platform is %s", actual) + } + return nil + }, + } +} diff --git a/pkg/e2e/volumes_test.go b/pkg/e2e/volumes_test.go index a9f5d46652..a5c7ee06d2 100644 --- a/pkg/e2e/volumes_test.go +++ b/pkg/e2e/volumes_test.go @@ -198,28 +198,29 @@ func TestImageVolumeImageAlreadyLocal(t *testing.T) { // digest the daemon can't resolve as a mount source under the containerd // image store ("No such image"). TestImageVolume only covers this path by // accident, when a previous test left the image in the local store. - c := NewCLI(t) - const projectName = "compose-e2e-image-volume-local" - t.Cleanup(func() { - c.cleanupWithDown(t, projectName) - }) - - version := c.RunDockerCmd(t, "version", "-f", "{{.Server.Version}}") - major, _, found := strings.Cut(version.Combined(), ".") - assert.Assert(t, found) - if major == "26" || major == "27" { - t.Skip("Skipping test due to docker version < 28") - } - - // make sure the source image is already in the local store - c.RunDockerCmd(t, "pull", "-q", "nginx:alpine") - - res := c.RunDockerComposeCmd(t, "-f", "./fixtures/volumes/compose.yaml", "--project-name", projectName, "up", "with_image") - assert.Check(t, strings.Contains(res.Combined(), "index.html"), res.Combined()) - - // the unchanged service must not be recreated by a second up - res = c.RunDockerComposeCmd(t, "-f", "./fixtures/volumes/compose.yaml", "--project-name", projectName, "up", "with_image") - assert.Check(t, !strings.Contains(res.Combined(), "Recreate"), res.Combined()) + NewScenario(t, "an image volume whose source image is already local must mount, and stay idempotent", Serial()). + Requires(EngineVersionAtLeast(28)). + Compose(` +services: + app: + image: alpine + command: "ls -al /mnt/image" + volumes: + - type: image + source: nginx:alpine + target: /mnt/image + image: + subpath: usr/share/nginx/html/ +`). + Step("have the source image already in the local store", + DockerCmd("pull", "-q", "nginx:alpine")). + Step("up mounts the image volume content", + ComposeCmd("up", "app"), + OutputContains("index.html")). + Step("an unchanged up does not recreate the service", + ComposeCmd("up", "app"), + OutputNotContains("Recreate"), + NotRecreated("app")) } func TestImageVolumeRecreateOnRebuild(t *testing.T) { From 8e88329c2616e19d3f746658247cd95445ef5bb1 Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Fri, 14 Aug 2026 17:11:22 +0200 Subject: [PATCH 2/5] test(e2e): Eventually check, TestRestart as polling demonstrator MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add Eventually(check, timeout) to the scenario layer: it retries a state-based check, re-observing the project containers between attempts. ServiceState(service, state) joins the check library, backed by a State field in the per-step snapshots. TestRestart is converted as the demonstrator: it replaces a WaitForCmdResult polling `ps --format json` for a JSON fragment and a bare time.Sleep with two declarative expectations, and now also locks the real invariant — restart must reuse the same container — via NotRecreated, which the lock-file trick of the original fixture was proving only indirectly. Signed-off-by: Nicolas De Loof --- pkg/e2e/restart_test.go | 46 +++++++++------------ pkg/e2e/scenario.go | 89 ++++++++++++++++++++++++++++++++++++++++- 2 files changed, 105 insertions(+), 30 deletions(-) diff --git a/pkg/e2e/restart_test.go b/pkg/e2e/restart_test.go index c9df28cb21..7ea2b4bfe2 100644 --- a/pkg/e2e/restart_test.go +++ b/pkg/e2e/restart_test.go @@ -34,34 +34,24 @@ func assertServiceStatus(t *testing.T, projectName, service, status string, ps s } func TestRestart(t *testing.T) { - c := NewParallelCLI(t) - const projectName = "e2e-restart" - - t.Run("Up a project", func(t *testing.T) { - // This is just to ensure the containers do NOT exist - c.RunDockerComposeCmd(t, "--project-name", projectName, "down") - - res := c.RunDockerComposeCmd(t, "-f", "./fixtures/restart-test/compose.yaml", "--project-name", projectName, "up", "-d") - assert.Assert(t, strings.Contains(res.Combined(), "Container e2e-restart-restart-1 Started"), res.Combined()) - - c.WaitForCmdResult(t, c.NewDockerComposeCmd(t, "--project-name", projectName, "ps", "-a", "--format", - "json"), - StdoutContains(`"State":"exited"`), 10*time.Second, 1*time.Second) - - res = c.RunDockerComposeCmd(t, "--project-name", projectName, "ps", "-a") - assertServiceStatus(t, projectName, "restart", "Exited", res.Stdout()) - - c.RunDockerComposeCmd(t, "-f", "./fixtures/restart-test/compose.yaml", "--project-name", projectName, "restart") - - // Give the same time but it must NOT exit - time.Sleep(time.Second) - - res = c.RunDockerComposeCmd(t, "--project-name", projectName, "ps") - assertServiceStatus(t, projectName, "restart", "Up", res.Stdout()) - - // Clean up - c.RunDockerComposeCmd(t, "--project-name", projectName, "down") - }) + // the service's first run creates a lock file and exits at once; any + // later run of the same container finds the lock and sleeps forever, so + // staying up after `restart` proves the same container was restarted + NewScenario(t, "restart must bring an exited service back up, restarting the same container"). + Compose(` +services: + app: + image: alpine + init: true + command: ash -c "if [[ -f /tmp/restart.lock ]] ; then sleep infinity; else touch /tmp/restart.lock; fi" +`). + Step("up starts the service, whose first run exits at once", + ComposeCmd("up", "-d"), + Eventually(ServiceState("app", "exited"), 10*time.Second)). + Step("restart brings the service back up, reusing the container", + ComposeCmd("restart"), + Eventually(ServiceState("app", "running"), 10*time.Second), + NotRecreated("app")) } func TestRestartWithDependencies(t *testing.T) { diff --git a/pkg/e2e/scenario.go b/pkg/e2e/scenario.go index 46b1a28a50..c13b5697a3 100644 --- a/pkg/e2e/scenario.go +++ b/pkg/e2e/scenario.go @@ -21,6 +21,7 @@ import ( "fmt" "os" "path/filepath" + "runtime" "slices" "strconv" "strings" @@ -29,6 +30,7 @@ import ( "unicode" "gotest.tools/v3/icmd" + "gotest.tools/v3/poll" ) // Scenario is a thin declarative layer over the e2e CLI helpers: a test reads @@ -65,6 +67,7 @@ type stepRecord struct { type containerState struct { ID string Name string + State string Labels map[string]string } @@ -259,8 +262,11 @@ func (s *Scenario) snapshot() snapshot { return snap } var containers []struct { - ID string `json:"Id"` - Name string `json:"Name"` + ID string `json:"Id"` + Name string `json:"Name"` + State struct { + Status string `json:"Status"` + } `json:"State"` Config struct { Labels map[string]string `json:"Labels"` } `json:"Config"` @@ -273,6 +279,7 @@ func (s *Scenario) snapshot() snapshot { snap[service] = append(snap[service], containerState{ ID: c.ID, Name: strings.TrimPrefix(c.Name, "/"), + State: c.State.Status, Labels: c.Config.Labels, }) } @@ -416,6 +423,15 @@ type CheckContext struct { curr snapshot } +// refresh re-observes the project state, so subsequent checks of the same +// step see the latest state rather than the one captured right after the +// command returned. +func (ctx *CheckContext) refresh() { + s := ctx.scenario + ctx.curr = s.snapshot() + s.snaps[len(s.snaps)-1] = ctx.curr +} + // Check is a named observable expected to hold after a step. type Check struct { name string @@ -450,6 +466,75 @@ func OutputNotContains(sub string) Check { } } +// Eventually retries a state-based check until it holds or the timeout +// expires, re-observing the project state between attempts. Output-based +// checks are not meaningful here: the step's output never changes. +// Polling is delegated to gotest.tools/v3/poll, the same engine the rest of +// the e2e framework uses. +func Eventually(check Check, timeout time.Duration) Check { + return Check{ + name: fmt.Sprintf("%s within %s", check.name, timeout), + fn: func(ctx *CheckContext) error { + first := true + capture := &pollCapture{} + done := make(chan struct{}) + // poll.WaitOn reports timeout through TestingT.Fatalf and relies + // on it halting execution; run it in a goroutine so pollCapture + // can Goexit and hand the error back to the scenario report + // instead of failing the test on the spot. + go func() { + defer close(done) + poll.WaitOn(capture, func(poll.LogT) poll.Result { + if !first { + ctx.refresh() + } + first = false + if err := check.fn(ctx); err != nil { + return poll.Continue("%v", err) + } + return poll.Success() + }, poll.WithDelay(500*time.Millisecond), poll.WithTimeout(timeout)) + }() + <-done + return capture.err + }, + } +} + +// pollCapture is a poll.TestingT that records the failure instead of failing +// the test, so Eventually can feed it to the scenario failure report. +type pollCapture struct { + err error +} + +func (c *pollCapture) Log(args ...any) {} +func (c *pollCapture) Logf(format string, args ...any) {} + +func (c *pollCapture) Fatalf(format string, args ...any) { + c.err = fmt.Errorf(format, args...) + runtime.Goexit() +} + +// ServiceState expects every container of the service to be in the given +// state (running, exited, restarting, …). +func ServiceState(service, state string) Check { + return Check{ + name: fmt.Sprintf("service %q is %s", service, state), + fn: func(ctx *CheckContext) error { + containers := ctx.curr[service] + if len(containers) == 0 { + return fmt.Errorf("service has no container") + } + for _, c := range containers { + if c.State != state { + return fmt.Errorf("container %s is %s", c.Name, c.State) + } + } + return nil + }, + } +} + // NotRecreated expects the services' containers to be exactly the ones that // existed before the step (same container IDs). func NotRecreated(services ...string) Check { From 24dfe42fd598eeb00a8474cb304b75cd74a0cb8e Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Sun, 16 Aug 2026 14:33:01 +0200 Subject: [PATCH 3/5] test(e2e): dump failure artifacts, E2E_KEEP_FAILED keeps the project alive On failure, a scenario now writes the untruncated diagnostic material to a stable per-project directory (compose.yaml, each step's full command and output, project containers, engine events, full container logs, per-step state snapshots), announced at the top of the failure report. The inline report stays truncated for readability; the artifacts let a failure be diagnosed without re-running the scenario. E2E_KEEP_FAILED=1 skips the teardown of a failed scenario so the live project state remains inspectable. Co-Authored-By: Claude Fable 5 Signed-off-by: Nicolas De Loof --- pkg/e2e/scenario.go | 93 +++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 86 insertions(+), 7 deletions(-) diff --git a/pkg/e2e/scenario.go b/pkg/e2e/scenario.go index c13b5697a3..96b877da03 100644 --- a/pkg/e2e/scenario.go +++ b/pkg/e2e/scenario.go @@ -110,6 +110,10 @@ func NewScenario(t *testing.T, intent string, opts ...ScenarioOption) *Scenario // start from — and return to — a clean slate, whatever previous runs left s.cli.RunDockerComposeCmdNoCheck(t, "--project-name", s.project, "down", "-v", "--remove-orphans", "--timeout", "0") t.Cleanup(func() { + if t.Failed() && os.Getenv("E2E_KEEP_FAILED") != "" { + t.Logf("E2E_KEEP_FAILED set: keeping project %s alive for inspection (docker ps --filter label=com.docker.compose.project=%s)", s.project, s.project) + return + } s.cli.RunDockerComposeCmdNoCheck(t, "--project-name", s.project, "down", "-v", "--remove-orphans", "--timeout", "0") for _, action := range s.deferred { _ = icmd.RunCmd(s.command(action)) @@ -300,13 +304,30 @@ func (s *Scenario) projectContainerIDs() []string { // fail reports the scenario failure: intent, step transcript, output of the // failing command, then live diagnostics (project state, engine events since -// the scenario started, container logs). +// the scenario started, container logs). Inline sections are truncated for +// readability; the full, untruncated material is written to an artifacts +// directory whose path opens the report. func (s *Scenario) fail(reason error) { t := s.t t.Helper() + + live := s.snapshot() + containersOut := s.diag("ps", "-a", "--no-trunc", "--filter", "label=com.docker.compose.project="+s.project) + eventsOut := s.diag("events", + "--since", s.start.Format(time.RFC3339Nano), "--until", time.Now().Format(time.RFC3339Nano), + "--filter", "label=com.docker.compose.project="+s.project) + artifacts := s.writeArtifacts(reason, live, containersOut, eventsOut) + var b strings.Builder fmt.Fprintf(&b, "scenario failed: %s\n", s.intent) - fmt.Fprintf(&b, "project: %s\n\ntranscript:\n", s.project) + fmt.Fprintf(&b, "project: %s\n", s.project) + if artifacts != "" { + fmt.Fprintf(&b, "artifacts: %s (compose.yaml, full step outputs, events, logs)\n", artifacts) + } + if os.Getenv("E2E_KEEP_FAILED") == "" { + fmt.Fprintf(&b, "hint: rerun with E2E_KEEP_FAILED=1 to keep the project alive for inspection\n") + } + fmt.Fprintf(&b, "\ntranscript:\n") for i, step := range s.steps { mark := "✓" if i == len(s.steps)-1 { @@ -317,11 +338,9 @@ func (s *Scenario) fail(reason error) { last := s.steps[len(s.steps)-1] fmt.Fprintf(&b, "\nfailure: %v\n\n--- output of failing step\n%s\n", reason, truncate(last.result.Combined(), 4000)) - fmt.Fprintf(&b, "\n--- project containers\n%s\n", truncate(s.diag("ps", "-a", "--no-trunc", "--filter", "label=com.docker.compose.project="+s.project), 2000)) - fmt.Fprintf(&b, "\n--- engine events since scenario start\n%s\n", truncate(s.diag("events", - "--since", s.start.Format(time.RFC3339Nano), "--until", time.Now().Format(time.RFC3339Nano), - "--filter", "label=com.docker.compose.project="+s.project), 2000)) - for _, containers := range s.snapshot() { + fmt.Fprintf(&b, "\n--- project containers\n%s\n", truncate(containersOut, 2000)) + fmt.Fprintf(&b, "\n--- engine events since scenario start\n%s\n", truncate(eventsOut, 2000)) + for _, containers := range live { for _, c := range containers { fmt.Fprintf(&b, "\n--- logs %s\n%s\n", c.Name, truncate(s.diag("logs", "--tail", "30", c.ID), 2000)) } @@ -329,6 +348,66 @@ func (s *Scenario) fail(reason error) { t.Fatal(b.String()) } +// writeArtifacts dumps the untruncated failure material to a stable directory +// (one per project, overwritten on each run) so a failure can be diagnosed — +// by a human or a coding agent — without re-running the scenario: the compose +// model, each step's full command and output, the project containers, the +// engine events, every container's full logs and the per-step state +// snapshots. Returns the directory path, or "" if it could not be written. +func (s *Scenario) writeArtifacts(reason error, live snapshot, containersOut, eventsOut string) string { + dir := filepath.Join(os.TempDir(), "compose-e2e-artifacts", s.project) + if err := os.RemoveAll(dir); err != nil { + return "" + } + if err := os.MkdirAll(dir, 0o755); err != nil { + return "" + } + write := func(name, content string) { + _ = os.WriteFile(filepath.Join(dir, name), []byte(content), 0o644) + } + + if s.file != "" { + if data, err := os.ReadFile(s.file); err == nil { + write("compose.yaml", string(data)) + } + } + write("failure.txt", fmt.Sprintf("scenario: %s\nproject: %s\nfailure: %v\n", s.intent, s.project, reason)) + for i, step := range s.steps { + write(fmt.Sprintf("step-%02d-%s.txt", i+1, slugify(step.name)), + fmt.Sprintf("step: %s\ncommand: %s\nexit code: %d\nduration: %s\n\n%s", + step.name, step.command, step.result.ExitCode, step.duration.Round(time.Millisecond), step.result.Combined())) + } + write("containers.txt", containersOut) + write("events.txt", eventsOut) + for _, containers := range live { + for _, c := range containers { + write("logs-"+c.Name+".txt", s.diag("logs", c.ID)) + } + } + if data, err := json.MarshalIndent(s.snaps, "", " "); err == nil { + write("snapshots.json", string(data)) + } + return dir +} + +// slugify turns a free-form step name into a safe file-name fragment. +func slugify(name string) string { + var b strings.Builder + pendingDash := false + for _, r := range strings.ToLower(name) { + if (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') { + if pendingDash && b.Len() > 0 { + b.WriteRune('-') + } + pendingDash = false + b.WriteRune(r) + } else { + pendingDash = true + } + } + return b.String() +} + func (s *Scenario) diag(args ...string) string { res := icmd.RunCmd(s.cli.NewDockerCmd(s.t, args...)) return strings.TrimSpace(res.Combined()) From a9c0af4babf807cd81a98cefa9b247a3a373c5d9 Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Sun, 16 Aug 2026 14:34:52 +0200 Subject: [PATCH 4/5] test(e2e): checks.go as the reviewed vocabulary, SCENARIO.md codifies the contract MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Checks are the shared vocabulary between scenarios: move them to a dedicated checks.go so the whole vocabulary is read — and reviewed — as one file, independently of the tests using it. SCENARIO.md codifies the rules that so far only lived in the PR description: one intent = one invariant, step names as behavior sentences, inline compose model, state-based checks first with OutputContains as an explicit last resort, new checks generic and named after their observable — plus how to exploit the failure artifacts and E2E_KEEP_FAILED. AGENTS.md points new e2e work at it. Co-Authored-By: Claude Fable 5 Signed-off-by: Nicolas De Loof --- AGENTS.md | 9 ++ pkg/e2e/SCENARIO.md | 91 +++++++++++++++ pkg/e2e/checks.go | 268 ++++++++++++++++++++++++++++++++++++++++++++ pkg/e2e/scenario.go | 238 --------------------------------------- 4 files changed, 368 insertions(+), 238 deletions(-) create mode 100644 pkg/e2e/SCENARIO.md create mode 100644 pkg/e2e/checks.go diff --git a/AGENTS.md b/AGENTS.md index a0f73f6825..e91c95f52a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -8,6 +8,15 @@ - Test single: `go test ./pkg/compose/ -run TestFunctionName` - E2E tests: `go test -tags e2e ./pkg/e2e/ -run TestName` +## E2E tests + +- **New e2e tests use the declarative `Scenario` DSL** (`NewScenario` in + `pkg/e2e/scenario.go`): intent, inline compose model, steps as + `(command → expected observables)`. Read `pkg/e2e/SCENARIO.md` before + writing or debugging one — it codifies the rules (state-based checks first, + `OutputContains` as last resort, new checks go in `pkg/e2e/checks.go`) and + how to exploit failure artifacts and `E2E_KEEP_FAILED=1`. + ## Lint - Linter: golangci-lint v2 (config in `.golangci.yml`) diff --git a/pkg/e2e/SCENARIO.md b/pkg/e2e/SCENARIO.md new file mode 100644 index 0000000000..96a8d5f6c2 --- /dev/null +++ b/pkg/e2e/SCENARIO.md @@ -0,0 +1,91 @@ +# E2E scenarios: the contract + +With coding agents writing most of the production code, e2e tests are the +document humans actually read and review. A scenario must state an intent, a +compose model, and a sequence of steps `(command → expected observables)` — +and nothing else. Everything operational (project naming, cleanup, failure +diagnostics) belongs to the framework, not to the test. + +The DSL lives in [`scenario.go`](scenario.go) (execution, actions, +requirements) and [`checks.go`](checks.go) (the vocabulary of observables). + +## Writing a scenario + +```go +func TestRestart(t *testing.T) { + NewScenario(t, "restart must bring an exited service back up, restarting the same container"). + Compose(` +services: + app: + image: alpine + init: true + command: ash -c "if [[ -f /tmp/restart.lock ]] ; then sleep infinity; else touch /tmp/restart.lock; fi" +`). + Step("up starts the service, whose first run exits at once", + ComposeCmd("up", "-d"), + Eventually(ServiceState("app", "exited"), 10*time.Second)). + Step("restart brings the service back up, reusing the container", + ComposeCmd("restart"), + Eventually(ServiceState("app", "running"), 10*time.Second), + NotRecreated("app")) +} +``` + +Rules: + +- **New e2e tests use `NewScenario`.** The legacy `NewCLI` style remains for + existing tests, converted opportunistically; don't add to it. +- **One intent = one invariant.** The intent is a one-line statement of the + behavior being locked, phrased as an obligation ("X must Y"). If you need + two intents, write two scenarios. +- **Step names are behavior sentences**, not command echoes: "an unchanged + create is a no-op", not "run create again". The transcript of step names + should read as the specification. +- **The compose model is inline.** A scenario is self-contained in the test + source; no fixture directories. Interpolate runtime values via `Env`. +- **Regression tests link the issue** in a comment above the test, with a + sentence on the failure mode being locked. + +## Checks: observe real state + +Checks are the shared vocabulary between scenarios; their discipline is what +keeps the contract meaningful. + +- **Prefer state-based checks** (`ServiceState`, `NotRecreated`, `LabelSet`, + `RunsOnPlatform`, …): they observe containers, labels and image manifests — + what the user actually gets — not what the CLI printed. +- **`OutputContains` is a last resort**, legitimate only when the CLI's + reported decision is itself the observable (e.g. "Skipped" vs "Pulled"). +- **Never poll by hand**: wrap a state check in `Eventually(check, timeout)`. + No `time.Sleep` in scenarios. +- **A new check must be generic** — no test-specific logic — **and named + after the observable it asserts**, not after the test that needed it. It + goes in `checks.go`, where the whole vocabulary is reviewed as one file. + Before adding one, verify the observable isn't already expressible. +- A check should also fail loudly on a broken precondition (e.g. + `NotRecreated` errors if the service had no container before the step) + rather than pass vacuously. + +## When a scenario fails + +The report opens with everything needed to diagnose without re-running: + +- **`artifacts: `** — a stable per-project directory holding the + untruncated material: `compose.yaml`, `failure.txt`, each step's full + command and output (`step-NN-*.txt`), `containers.txt`, `events.txt`, full + container logs (`logs-*.txt`) and the per-step state snapshots + (`snapshots.json`). Read these before re-running anything. +- **`E2E_KEEP_FAILED=1`** — rerun with this set to skip teardown of failed + scenarios: containers, volumes and networks stay alive for `docker + inspect`/`exec`. Clean up afterwards with + `docker compose --project-name down -v --remove-orphans`. +- The inline report shows the transcript (every step, exit code, duration), + the failing step's output, project containers, engine events since the + scenario started, and container log tails — truncated for readability; the + artifacts have the full versions. + +Run a single scenario with: + +```sh +go test -tags e2e ./pkg/e2e/ -run TestRestart -v +``` diff --git a/pkg/e2e/checks.go b/pkg/e2e/checks.go new file mode 100644 index 0000000000..88a6bb0f8a --- /dev/null +++ b/pkg/e2e/checks.go @@ -0,0 +1,268 @@ +/* + Copyright 2026 Docker Compose CLI authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package e2e + +// This file is the scenario vocabulary: the complete set of observables a +// step can expect. It is meant to be read — and reviewed — as a whole; see +// SCENARIO.md for the rules governing its growth. In short: checks observe +// real state (containers, labels, image manifests), are generic (no +// test-specific logic), and are named after the observable they assert. + +import ( + "fmt" + "runtime" + "slices" + "strings" + "time" + + "gotest.tools/v3/icmd" + "gotest.tools/v3/poll" +) + +// CheckContext gives a check access to the step's result and to the project +// state captured before and after the step. +type CheckContext struct { + scenario *Scenario + result *icmd.Result + prev snapshot + curr snapshot +} + +// refresh re-observes the project state, so subsequent checks of the same +// step see the latest state rather than the one captured right after the +// command returned. +func (ctx *CheckContext) refresh() { + s := ctx.scenario + ctx.curr = s.snapshot() + s.snaps[len(s.snaps)-1] = ctx.curr +} + +// Check is a named observable expected to hold after a step. +type Check struct { + name string + fn func(*CheckContext) error +} + +// OutputContains expects the command output to contain a string. Prefer +// state-based checks; use this when the CLI's reported decision is itself +// the observable. +func OutputContains(sub string) Check { + return Check{ + name: fmt.Sprintf("output contains %q", sub), + fn: func(ctx *CheckContext) error { + if !strings.Contains(ctx.result.Combined(), sub) { + return fmt.Errorf("not found in output") + } + return nil + }, + } +} + +// OutputNotContains expects the command output not to contain a string. +func OutputNotContains(sub string) Check { + return Check{ + name: fmt.Sprintf("output does not contain %q", sub), + fn: func(ctx *CheckContext) error { + if strings.Contains(ctx.result.Combined(), sub) { + return fmt.Errorf("found in output") + } + return nil + }, + } +} + +// Eventually retries a state-based check until it holds or the timeout +// expires, re-observing the project state between attempts. Output-based +// checks are not meaningful here: the step's output never changes. +// Polling is delegated to gotest.tools/v3/poll, the same engine the rest of +// the e2e framework uses. +func Eventually(check Check, timeout time.Duration) Check { + return Check{ + name: fmt.Sprintf("%s within %s", check.name, timeout), + fn: func(ctx *CheckContext) error { + first := true + capture := &pollCapture{} + done := make(chan struct{}) + // poll.WaitOn reports timeout through TestingT.Fatalf and relies + // on it halting execution; run it in a goroutine so pollCapture + // can Goexit and hand the error back to the scenario report + // instead of failing the test on the spot. + go func() { + defer close(done) + poll.WaitOn(capture, func(poll.LogT) poll.Result { + if !first { + ctx.refresh() + } + first = false + if err := check.fn(ctx); err != nil { + return poll.Continue("%v", err) + } + return poll.Success() + }, poll.WithDelay(500*time.Millisecond), poll.WithTimeout(timeout)) + }() + <-done + return capture.err + }, + } +} + +// pollCapture is a poll.TestingT that records the failure instead of failing +// the test, so Eventually can feed it to the scenario failure report. +type pollCapture struct { + err error +} + +func (c *pollCapture) Log(args ...any) {} +func (c *pollCapture) Logf(format string, args ...any) {} + +func (c *pollCapture) Fatalf(format string, args ...any) { + c.err = fmt.Errorf(format, args...) + runtime.Goexit() +} + +// ServiceState expects every container of the service to be in the given +// state (running, exited, restarting, …). +func ServiceState(service, state string) Check { + return Check{ + name: fmt.Sprintf("service %q is %s", service, state), + fn: func(ctx *CheckContext) error { + containers := ctx.curr[service] + if len(containers) == 0 { + return fmt.Errorf("service has no container") + } + for _, c := range containers { + if c.State != state { + return fmt.Errorf("container %s is %s", c.Name, c.State) + } + } + return nil + }, + } +} + +// NotRecreated expects the services' containers to be exactly the ones that +// existed before the step (same container IDs). +func NotRecreated(services ...string) Check { + return Check{ + name: fmt.Sprintf("services %s not recreated", strings.Join(services, ", ")), + fn: func(ctx *CheckContext) error { + for _, service := range services { + before, after := containerIDs(ctx.prev[service]), containerIDs(ctx.curr[service]) + if len(before) == 0 { + return fmt.Errorf("service %q had no container before the step", service) + } + if !slices.Equal(before, after) { + return fmt.Errorf("service %q containers changed: %v -> %v", service, before, after) + } + } + return nil + }, + } +} + +func containerIDs(containers []containerState) []string { + ids := make([]string, 0, len(containers)) + for _, c := range containers { + ids = append(ids, c.ID) + } + slices.Sort(ids) + return ids +} + +// LabelSet expects every container of the service to carry a non-empty label. +func LabelSet(service, key string) Check { + return Check{ + name: fmt.Sprintf("service %q has label %q set", service, key), + fn: func(ctx *CheckContext) error { + containers := ctx.curr[service] + if len(containers) == 0 { + return fmt.Errorf("service has no container") + } + for _, c := range containers { + if c.Labels[key] == "" { + return fmt.Errorf("label empty on container %s", c.Name) + } + } + return nil + }, + } +} + +// LabelsDistinct expects the given services to carry pairwise-distinct values +// for a label. +func LabelsDistinct(key string, services ...string) Check { + return Check{ + name: fmt.Sprintf("services %s have distinct %q labels", strings.Join(services, ", "), key), + fn: func(ctx *CheckContext) error { + seen := map[string]string{} + for _, service := range services { + containers := ctx.curr[service] + if len(containers) == 0 { + return fmt.Errorf("service %q has no container", service) + } + value := containers[0].Labels[key] + if other, dup := seen[value]; dup { + return fmt.Errorf("services %q and %q share label value %q", other, service, value) + } + seen[value] = service + } + return nil + }, + } +} + +// LabelUnchanged expects a service's label to have the same value as before +// the step. +func LabelUnchanged(service, key string) Check { + return Check{ + name: fmt.Sprintf("service %q label %q unchanged", service, key), + fn: func(ctx *CheckContext) error { + before, after := ctx.prev[service], ctx.curr[service] + if len(before) == 0 || len(after) == 0 { + return fmt.Errorf("service has no container to compare") + } + if before[0].Labels[key] != after[0].Labels[key] { + return fmt.Errorf("label changed: %q -> %q", before[0].Labels[key], after[0].Labels[key]) + } + return nil + }, + } +} + +// RunsOnPlatform expects the service's container to have been created for +// the given platform (from its image manifest descriptor). +func RunsOnPlatform(service, platform string) Check { + return Check{ + name: fmt.Sprintf("service %q container created for platform %s", service, platform), + fn: func(ctx *CheckContext) error { + containers := ctx.curr[service] + if len(containers) == 0 { + return fmt.Errorf("service has no container") + } + res := icmd.RunCmd(ctx.scenario.cli.NewDockerCmd(ctx.scenario.t, "inspect", "--format", + "{{.ImageManifestDescriptor.Platform.OS}}/{{.ImageManifestDescriptor.Platform.Architecture}}", + containers[0].ID)) + if res.ExitCode != 0 { + return fmt.Errorf("inspect failed: %s", res.Combined()) + } + if actual := strings.TrimSpace(res.Stdout()); actual != platform { + return fmt.Errorf("container platform is %s", actual) + } + return nil + }, + } +} diff --git a/pkg/e2e/scenario.go b/pkg/e2e/scenario.go index 96b877da03..b38e6bf3c9 100644 --- a/pkg/e2e/scenario.go +++ b/pkg/e2e/scenario.go @@ -21,7 +21,6 @@ import ( "fmt" "os" "path/filepath" - "runtime" "slices" "strconv" "strings" @@ -30,7 +29,6 @@ import ( "unicode" "gotest.tools/v3/icmd" - "gotest.tools/v3/poll" ) // Scenario is a thin declarative layer over the e2e CLI helpers: a test reads @@ -490,239 +488,3 @@ func EngineVersionAtLeast(major int) Requirement { return "" } } - -// ---------- Checks ---------- - -// CheckContext gives a check access to the step's result and to the project -// state captured before and after the step. -type CheckContext struct { - scenario *Scenario - result *icmd.Result - prev snapshot - curr snapshot -} - -// refresh re-observes the project state, so subsequent checks of the same -// step see the latest state rather than the one captured right after the -// command returned. -func (ctx *CheckContext) refresh() { - s := ctx.scenario - ctx.curr = s.snapshot() - s.snaps[len(s.snaps)-1] = ctx.curr -} - -// Check is a named observable expected to hold after a step. -type Check struct { - name string - fn func(*CheckContext) error -} - -// OutputContains expects the command output to contain a string. Prefer -// state-based checks; use this when the CLI's reported decision is itself -// the observable. -func OutputContains(sub string) Check { - return Check{ - name: fmt.Sprintf("output contains %q", sub), - fn: func(ctx *CheckContext) error { - if !strings.Contains(ctx.result.Combined(), sub) { - return fmt.Errorf("not found in output") - } - return nil - }, - } -} - -// OutputNotContains expects the command output not to contain a string. -func OutputNotContains(sub string) Check { - return Check{ - name: fmt.Sprintf("output does not contain %q", sub), - fn: func(ctx *CheckContext) error { - if strings.Contains(ctx.result.Combined(), sub) { - return fmt.Errorf("found in output") - } - return nil - }, - } -} - -// Eventually retries a state-based check until it holds or the timeout -// expires, re-observing the project state between attempts. Output-based -// checks are not meaningful here: the step's output never changes. -// Polling is delegated to gotest.tools/v3/poll, the same engine the rest of -// the e2e framework uses. -func Eventually(check Check, timeout time.Duration) Check { - return Check{ - name: fmt.Sprintf("%s within %s", check.name, timeout), - fn: func(ctx *CheckContext) error { - first := true - capture := &pollCapture{} - done := make(chan struct{}) - // poll.WaitOn reports timeout through TestingT.Fatalf and relies - // on it halting execution; run it in a goroutine so pollCapture - // can Goexit and hand the error back to the scenario report - // instead of failing the test on the spot. - go func() { - defer close(done) - poll.WaitOn(capture, func(poll.LogT) poll.Result { - if !first { - ctx.refresh() - } - first = false - if err := check.fn(ctx); err != nil { - return poll.Continue("%v", err) - } - return poll.Success() - }, poll.WithDelay(500*time.Millisecond), poll.WithTimeout(timeout)) - }() - <-done - return capture.err - }, - } -} - -// pollCapture is a poll.TestingT that records the failure instead of failing -// the test, so Eventually can feed it to the scenario failure report. -type pollCapture struct { - err error -} - -func (c *pollCapture) Log(args ...any) {} -func (c *pollCapture) Logf(format string, args ...any) {} - -func (c *pollCapture) Fatalf(format string, args ...any) { - c.err = fmt.Errorf(format, args...) - runtime.Goexit() -} - -// ServiceState expects every container of the service to be in the given -// state (running, exited, restarting, …). -func ServiceState(service, state string) Check { - return Check{ - name: fmt.Sprintf("service %q is %s", service, state), - fn: func(ctx *CheckContext) error { - containers := ctx.curr[service] - if len(containers) == 0 { - return fmt.Errorf("service has no container") - } - for _, c := range containers { - if c.State != state { - return fmt.Errorf("container %s is %s", c.Name, c.State) - } - } - return nil - }, - } -} - -// NotRecreated expects the services' containers to be exactly the ones that -// existed before the step (same container IDs). -func NotRecreated(services ...string) Check { - return Check{ - name: fmt.Sprintf("services %s not recreated", strings.Join(services, ", ")), - fn: func(ctx *CheckContext) error { - for _, service := range services { - before, after := containerIDs(ctx.prev[service]), containerIDs(ctx.curr[service]) - if len(before) == 0 { - return fmt.Errorf("service %q had no container before the step", service) - } - if !slices.Equal(before, after) { - return fmt.Errorf("service %q containers changed: %v -> %v", service, before, after) - } - } - return nil - }, - } -} - -func containerIDs(containers []containerState) []string { - ids := make([]string, 0, len(containers)) - for _, c := range containers { - ids = append(ids, c.ID) - } - slices.Sort(ids) - return ids -} - -// LabelSet expects every container of the service to carry a non-empty label. -func LabelSet(service, key string) Check { - return Check{ - name: fmt.Sprintf("service %q has label %q set", service, key), - fn: func(ctx *CheckContext) error { - containers := ctx.curr[service] - if len(containers) == 0 { - return fmt.Errorf("service has no container") - } - for _, c := range containers { - if c.Labels[key] == "" { - return fmt.Errorf("label empty on container %s", c.Name) - } - } - return nil - }, - } -} - -// LabelsDistinct expects the given services to carry pairwise-distinct values -// for a label. -func LabelsDistinct(key string, services ...string) Check { - return Check{ - name: fmt.Sprintf("services %s have distinct %q labels", strings.Join(services, ", "), key), - fn: func(ctx *CheckContext) error { - seen := map[string]string{} - for _, service := range services { - containers := ctx.curr[service] - if len(containers) == 0 { - return fmt.Errorf("service %q has no container", service) - } - value := containers[0].Labels[key] - if other, dup := seen[value]; dup { - return fmt.Errorf("services %q and %q share label value %q", other, service, value) - } - seen[value] = service - } - return nil - }, - } -} - -// LabelUnchanged expects a service's label to have the same value as before -// the step. -func LabelUnchanged(service, key string) Check { - return Check{ - name: fmt.Sprintf("service %q label %q unchanged", service, key), - fn: func(ctx *CheckContext) error { - before, after := ctx.prev[service], ctx.curr[service] - if len(before) == 0 || len(after) == 0 { - return fmt.Errorf("service has no container to compare") - } - if before[0].Labels[key] != after[0].Labels[key] { - return fmt.Errorf("label changed: %q -> %q", before[0].Labels[key], after[0].Labels[key]) - } - return nil - }, - } -} - -// RunsOnPlatform expects the service's container to have been created for -// the given platform (from its image manifest descriptor). -func RunsOnPlatform(service, platform string) Check { - return Check{ - name: fmt.Sprintf("service %q container created for platform %s", service, platform), - fn: func(ctx *CheckContext) error { - containers := ctx.curr[service] - if len(containers) == 0 { - return fmt.Errorf("service has no container") - } - res := icmd.RunCmd(ctx.scenario.cli.NewDockerCmd(ctx.scenario.t, "inspect", "--format", - "{{.ImageManifestDescriptor.Platform.OS}}/{{.ImageManifestDescriptor.Platform.Architecture}}", - containers[0].ID)) - if res.ExitCode != 0 { - return fmt.Errorf("inspect failed: %s", res.Combined()) - } - if actual := strings.TrimSpace(res.Stdout()); actual != platform { - return fmt.Errorf("container platform is %s", actual) - } - return nil - }, - } -} From 0ac224d9d9ff552f61042f593b3da6f49da76ede Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Sun, 16 Aug 2026 17:10:06 +0200 Subject: [PATCH 5/5] test(e2e): Files() declares multi-file projects as a txtar archive MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A scenario could only declare a lone compose.yaml; anything needing a Dockerfile, an env file or a config file fell back to fixture directories, breaking the "self-contained in the test source" rule. Files() takes the project files as a txtar archive (golang.org/x/tools/txtar, already in the module graph): each file introduced by a `-- name --` line, extracted into the project directory. txtar is the format Go's own cmd/go tests are written in — diff-friendly, and one every human and coding agent already knows by heart, which is the point: e2e scenarios are the main reading surface between the two. TestUpBuildUnchangedContext demonstrates the need — compose.yaml + Dockerfile + a copied file — and locks a real invariant: an unchanged `up --build` hits the build cache and must not recreate the service. Project() is exposed so a scenario can Defer removal of the image it built. Co-Authored-By: Claude Fable 5 Signed-off-by: Nicolas De Loof --- go.mod | 1 + pkg/e2e/SCENARIO.md | 18 +++++++++++++++++- pkg/e2e/build_test.go | 27 +++++++++++++++++++++++++++ pkg/e2e/scenario.go | 32 ++++++++++++++++++++++++++++++++ 4 files changed, 77 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index bd1f292cd7..a4c579195f 100644 --- a/go.mod +++ b/go.mod @@ -55,6 +55,7 @@ require ( go.yaml.in/yaml/v4 v4.0.0-rc.6 golang.org/x/sync v0.22.0 golang.org/x/sys v0.47.0 + golang.org/x/tools v0.47.0 google.golang.org/grpc v1.83.0 gotest.tools/v3 v3.5.2 tags.cncf.io/container-device-interface v1.1.0 diff --git a/pkg/e2e/SCENARIO.md b/pkg/e2e/SCENARIO.md index 96a8d5f6c2..3b5ac24651 100644 --- a/pkg/e2e/SCENARIO.md +++ b/pkg/e2e/SCENARIO.md @@ -42,7 +42,23 @@ Rules: create is a no-op", not "run create again". The transcript of step names should read as the specification. - **The compose model is inline.** A scenario is self-contained in the test - source; no fixture directories. Interpolate runtime values via `Env`. + source; no fixture directories. Interpolate runtime values via `Env`. When + the project needs more than a `compose.yaml` (a Dockerfile, an env file, a + config file), declare all files with `Files` as a [txtar](https://pkg.go.dev/golang.org/x/tools/txtar) + archive — the format Go's own `cmd/go` tests are written in: diff-friendly, + and one every human and coding agent already knows by heart: + + ```go + s.Files(` + -- compose.yaml -- + services: + app: + build: . + -- Dockerfile -- + FROM alpine + CMD ["sleep", "infinity"] + `) + ``` - **Regression tests link the issue** in a comment above the test, with a sentence on the failure mode being locked. diff --git a/pkg/e2e/build_test.go b/pkg/e2e/build_test.go index d917336ba0..4ae48634c3 100644 --- a/pkg/e2e/build_test.go +++ b/pkg/e2e/build_test.go @@ -673,3 +673,30 @@ func TestBuildEscaped(t *testing.T) { res = c.RunDockerComposeCmd(t, "--project-directory", "./fixtures/build-test/escaped", "build", "--no-cache", "arg") res.Assert(t, icmd.Success) } + +// TestUpBuildUnchangedContext locks the invariant that rebuilding an +// unchanged build context hits the build cache and leaves the running +// service alone: same image, same config hash, same container. +func TestUpBuildUnchangedContext(t *testing.T) { + s := NewScenario(t, "an unchanged up --build must hit the build cache and not recreate the service") + s.Files(` +-- compose.yaml -- +services: + app: + build: . +-- Dockerfile -- +FROM alpine +COPY marker /marker +CMD ["sleep", "infinity"] +-- marker -- +v1 +`). + Defer(DockerCmd("image", "rm", "-f", s.Project()+"-app")). + Step("up builds the image and starts the service", + ComposeCmd("up", "-d", "--build"), + ServiceState("app", "running")). + Step("an unchanged up --build reuses the cached image and the container", + ComposeCmd("up", "-d", "--build"), + NotRecreated("app"), + LabelUnchanged("app", "com.docker.compose.config-hash")) +} diff --git a/pkg/e2e/scenario.go b/pkg/e2e/scenario.go index b38e6bf3c9..8734407be8 100644 --- a/pkg/e2e/scenario.go +++ b/pkg/e2e/scenario.go @@ -28,6 +28,7 @@ import ( "time" "unicode" + "golang.org/x/tools/txtar" "gotest.tools/v3/icmd" ) @@ -147,6 +148,10 @@ func projectNameFor(testName string) string { // declarative layer doesn't cover. func (s *Scenario) CLI() *CLI { return s.cli } +// Project returns the compose project name the scenario runs under, e.g. to +// Defer the removal of an image the project built. +func (s *Scenario) Project() string { return s.project } + // Compose declares the project's compose model, written to a temporary // directory so the whole scenario is self-contained in the test source. func (s *Scenario) Compose(yaml string) *Scenario { @@ -159,6 +164,33 @@ func (s *Scenario) Compose(yaml string) *Scenario { return s } +// Files declares the project's files — compose.yaml plus whatever it needs +// (Dockerfile, .env, config files) — as a txtar archive: each file introduced +// by a `-- name --` line, extracted into the project directory. The archive +// must contain a compose.yaml, which becomes the scenario's compose file. +// txtar is the format Go's own cmd/go tests are written in: diff-friendly, +// and trivial to read and write for humans and coding agents alike. +func (s *Scenario) Files(archive string) *Scenario { + s.t.Helper() + dir := s.t.TempDir() + for _, f := range txtar.Parse([]byte(archive)).Files { + path := filepath.Join(dir, f.Name) + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + s.t.Fatalf("failed to create directory for %s: %v", f.Name, err) + } + if err := os.WriteFile(path, f.Data, 0o644); err != nil { + s.t.Fatalf("failed to write %s: %v", f.Name, err) + } + if f.Name == "compose.yaml" { + s.file = path + } + } + if s.file == "" { + s.t.Fatal("Files archive must contain a compose.yaml") + } + return s +} + // Env sets environment variables applied to every subsequent step command // (and interpolated in the compose model). func (s *Scenario) Env(kv ...string) *Scenario {