feat: differentiated exit codes (0/1/2/3/4) for the Kosli CLI

TODO.md

@@ -3,6 +3,13 @@
 <!-- Each feature gets its own ## section below. -->
 <!-- Only edit YOUR feature's section. Delete it after merging to main. -->
 
+## Structured exit codes (issue #4988)
+
+- [x] Slice 1: Typed errors + `os.Exit` dispatch in `main.go` + compliance exit code 1
+- [x] Slice 2: HTTP error classification in `requests.go` (5xx→2, 401/403→3, network→2)
+- [x] Slice 3: Usage errors (unknown flag, missing required flag → exit 4)
+- [x] Slice 4: Remaining assert commands + docs
+
 ## kosli evaluate trail
 
 - [x] Slice 1: Skeleton `evaluate` parent + `evaluate trail` fetches trail JSON

cmd/kosli/assertApproval.go

@@ -7,6 +7,7 @@ import (
 	"net/http"
 	"net/url"
 
+	kosliErrors "github.com/kosli-dev/cli/internal/errors"
 	"github.com/kosli-dev/cli/internal/requests"
 	"github.com/spf13/cobra"
 )
@@ -116,7 +117,7 @@ func (o *assertApprovalOptions) run(args []string) error {
  return err
 	}
 	if len(approvals) == 0 {
- return fmt.Errorf("artifact with fingerprint %s has no approvals created", o.fingerprint)
+ return kosliErrors.NewErrCompliance(fmt.Sprintf("artifact with fingerprint %s has no approvals created", o.fingerprint))
 	}
 
 	state, ok := approvals[len(approvals)-1]["state"].(string)
@@ -125,6 +126,6 @@ func (o *assertApprovalOptions) run(args []string) error {
  logger.Info("artifact with fingerprint %s is approved (approval no. [%v])", o.fingerprint, approvalNumber)
  return nil
 	} else {
- return fmt.Errorf("artifact with fingerprint %s is not approved", o.fingerprint)
+ return kosliErrors.NewErrCompliance(fmt.Sprintf("artifact with fingerprint %s is not approved", o.fingerprint))
 	}
 }

cmd/kosli/assertApproval_test.go

@@ -49,10 +49,11 @@ func (suite *AssertApprovalCommandTestSuite) SetupTest() {
 func (suite *AssertApprovalCommandTestSuite) TestAssertApprovalCmd() {
 	tests := []cmdTestCase{
  {
- wantError: true,
- name: "1 missing --org fails",
- cmd: fmt.Sprintf(`assert approval --fingerprint 8e568bd886069f1290def0caabc1e97ce0e7b80c105e611258b57d76fcef234c --flow %s --api-token secret`, suite.flowName),
- golden: "Error: --org is not set\nUsage: kosli assert approval [IMAGE-NAME | FILE-PATH | DIR-PATH] [flags]\n",
+ wantError: true,
+ wantExitCode: 4,
+ name: "1 missing --org fails",
+ cmd: fmt.Sprintf(`assert approval --fingerprint 8e568bd886069f1290def0caabc1e97ce0e7b80c105e611258b57d76fcef234c --flow %s --api-token secret`, suite.flowName),
+ golden: "Error: --org is not set\nUsage: kosli assert approval [IMAGE-NAME | FILE-PATH | DIR-PATH] [flags]\n",
  },
  {
  wantError: true,
@@ -61,21 +62,24 @@ func (suite *AssertApprovalCommandTestSuite) TestAssertApprovalCmd() {
  golden: "Error: Artifact with fingerprint '8e568bd886069f1290def0caabc1e97ce0e7b80c105e611258b57d76fcef234c' does not exist in flow 'assert-approval' belonging to organization 'docs-cmd-test-user'\n",
  },
  {
- wantError: true,
- name: "3 asserting an existing artifact that does not have an approval (using --fingerprint) works and exits with non-zero code",
- cmd: fmt.Sprintf(`assert approval --fingerprint %s --flow %s %s`, suite.fingerprint, suite.flowName, suite.defaultKosliArguments),
- golden: "Error: artifact with fingerprint fcf33337634c2577a5d86fd7ecb0a25a7c1bb5d89c14fd236f546a5759252c02 has no approvals created\n",
+ wantError: true,
+ wantExitCode: 1,
+ name: "3 asserting an existing artifact that does not have an approval (using --fingerprint) works and exits with non-zero code",
+ cmd: fmt.Sprintf(`assert approval --fingerprint %s --flow %s %s`, suite.fingerprint, suite.flowName, suite.defaultKosliArguments),
+ golden: "Error: artifact with fingerprint fcf33337634c2577a5d86fd7ecb0a25a7c1bb5d89c14fd236f546a5759252c02 has no approvals created\n",
  },
  {
- wantError: true,
- name: "4 asserting approval of an existing artifact that does not have an approval (using --artifact-type) works and exits with non-zero code",
- cmd: fmt.Sprintf(`assert approval %s --artifact-type file --flow %s %s`, suite.artifactPath, suite.flowName, suite.defaultKosliArguments),
- golden: "Error: artifact with fingerprint fcf33337634c2577a5d86fd7ecb0a25a7c1bb5d89c14fd236f546a5759252c02 has no approvals created\n",
+ wantError: true,
+ wantExitCode: 1,
+ name: "4 asserting approval of an existing artifact that does not have an approval (using --artifact-type) works and exits with non-zero code",
+ cmd: fmt.Sprintf(`assert approval %s --artifact-type file --flow %s %s`, suite.artifactPath, suite.flowName, suite.defaultKosliArguments),
+ golden: "Error: artifact with fingerprint fcf33337634c2577a5d86fd7ecb0a25a7c1bb5d89c14fd236f546a5759252c02 has no approvals created\n",
  },
  {
- name: "5 asserting approval of an existing artifact that has an approval (using --artifact-type) works and exits with zero code",
- cmd: fmt.Sprintf(`assert approval %s --artifact-type file --flow %s %s`, suite.artifactPath, suite.flowName, suite.defaultKosliArguments),
- golden: "artifact with fingerprint fcf33337634c2577a5d86fd7ecb0a25a7c1bb5d89c14fd236f546a5759252c02 is approved (approval no. [1])\n",
+ name: "5 asserting approval of an existing artifact that has an approval (using --artifact-type) works and exits with zero code",
+ wantExitCode: 0,
+ cmd: fmt.Sprintf(`assert approval %s --artifact-type file --flow %s %s`, suite.artifactPath, suite.flowName, suite.defaultKosliArguments),
+ golden: "artifact with fingerprint fcf33337634c2577a5d86fd7ecb0a25a7c1bb5d89c14fd236f546a5759252c02 is approved (approval no. [1])\n",
  additionalConfig: assertApprovalTestConfig{
  createApproval: true,
  isRequest: false,
@@ -88,39 +92,44 @@ func (suite *AssertApprovalCommandTestSuite) TestAssertApprovalCmd() {
  golden: "artifact with fingerprint fcf33337634c2577a5d86fd7ecb0a25a7c1bb5d89c14fd236f546a5759252c02 is approved (approval no. [1])\n",
  },
  {
- wantError: true,
- name: "7 not providing --fingerprint nor --artifact-type fails",
- cmd: fmt.Sprintf(`assert approval --flow %s %s`, suite.flowName, suite.defaultKosliArguments),
- golden: "Error: docker image name or file/dir path is required when --fingerprint is not provided\nUsage: kosli assert approval [IMAGE-NAME | FILE-PATH | DIR-PATH] [flags]\n",
+ wantError: true,
+ wantExitCode: 4,
+ name: "7 not providing --fingerprint nor --artifact-type fails",
+ cmd: fmt.Sprintf(`assert approval --flow %s %s`, suite.flowName, suite.defaultKosliArguments),
+ golden: "Error: docker image name or file/dir path is required when --fingerprint is not provided\nUsage: kosli assert approval [IMAGE-NAME | FILE-PATH | DIR-PATH] [flags]\n",
  },
  {
- wantError: true,
- name: "8 providing both --fingerprint and --artifact-type fails",
- cmd: fmt.Sprintf(`assert approval --artifact-type file --fingerprint %s --flow %s %s`, suite.fingerprint, suite.flowName, suite.defaultKosliArguments),
- golden: "Error: only one of --fingerprint, --artifact-type is allowed\n",
+ wantError: true,
+ wantExitCode: 1,
+ name: "8 providing both --fingerprint and --artifact-type fails",
+ cmd: fmt.Sprintf(`assert approval --artifact-type file --fingerprint %s --flow %s %s`, suite.fingerprint, suite.flowName, suite.defaultKosliArguments),
+ golden: "Error: only one of --fingerprint, --artifact-type is allowed\n",
  },
  {
- wantError: true,
- name: "9 missing --flow fails",
- cmd: fmt.Sprintf(`assert approval --fingerprint %s %s`, suite.fingerprint, suite.defaultKosliArguments),
- golden: "Error: required flag(s) \"flow\" not set\n",
+ wantError: true,
+ wantExitCode: 4,
+ name: "9 missing --flow fails",
+ cmd: fmt.Sprintf(`assert approval --fingerprint %s %s`, suite.fingerprint, suite.defaultKosliArguments),
+ golden: "Error: required flag(s) \"flow\" not set\n",
  },
  {
- wantError: true,
- name: "10 asserting approval of an unapproved existing artifact (using --artifact-type) works and exits with non-zero code",
- cmd: fmt.Sprintf(`assert approval %s --artifact-type file --flow %s %s`, suite.artifactPath, suite.flowName, suite.defaultKosliArguments),
- golden: "Error: artifact with fingerprint fcf33337634c2577a5d86fd7ecb0a25a7c1bb5d89c14fd236f546a5759252c02 is not approved\n",
+ wantError: true,
+ wantExitCode: 1,
+ name: "10 asserting approval of an unapproved existing artifact (using --artifact-type) works and exits with non-zero code",
+ cmd: fmt.Sprintf(`assert approval %s --artifact-type file --flow %s %s`, suite.artifactPath, suite.flowName, suite.defaultKosliArguments),
+ golden: "Error: artifact with fingerprint fcf33337634c2577a5d86fd7ecb0a25a7c1bb5d89c14fd236f546a5759252c02 is not approved\n",
  additionalConfig: assertApprovalTestConfig{
  createApproval: true,
  isRequest: true,
  },
  },
  // The approval request created in test 9 is valid here too
  {
- wantError: true,
- name: "11 asserting approval of an unapproved existing artifact (using --fingerprint) works and exits with non-zero code",
- cmd: fmt.Sprintf(`assert approval --fingerprint %s --flow %s %s`, suite.fingerprint, suite.flowName, suite.defaultKosliArguments),
- golden: "Error: artifact with fingerprint fcf33337634c2577a5d86fd7ecb0a25a7c1bb5d89c14fd236f546a5759252c02 is not approved\n",
+ wantError: true,
+ wantExitCode: 1,
+ name: "11 asserting approval of an unapproved existing artifact (using --fingerprint) works and exits with non-zero code",
+ cmd: fmt.Sprintf(`assert approval --fingerprint %s --flow %s %s`, suite.fingerprint, suite.flowName, suite.defaultKosliArguments),
+ golden: "Error: artifact with fingerprint fcf33337634c2577a5d86fd7ecb0a25a7c1bb5d89c14fd236f546a5759252c02 is not approved\n",
  },
 	}
 

cmd/kosli/assertArtifact.go

@@ -7,6 +7,7 @@ import (
 	"net/http"
 	"net/url"
 
+	kosliErrors "github.com/kosli-dev/cli/internal/errors"
 	"github.com/kosli-dev/cli/internal/output"
 	"github.com/kosli-dev/cli/internal/requests"
 	"github.com/spf13/cobra"
@@ -169,7 +170,7 @@ func (o *assertArtifactOptions) run(out io.Writer, args []string) error {
 
 	isCompliant := evaluationResult["compliant"].(bool)
 	if !isCompliant {
- return fmt.Errorf("Artifact is not compliant")
+ return kosliErrors.NewErrCompliance("Artifact is not compliant")
 	}
 	return nil
 }

cmd/kosli/assertArtifact_test.go

@@ -83,10 +83,11 @@ func (suite *AssertArtifactCommandTestSuite) SetupTest() {
 func (suite *AssertArtifactCommandTestSuite) TestAssertArtifactCmd() {
 	tests := []cmdTestCase{
  {
- wantError: true,
- name: "01 missing --org fails",
- cmd: fmt.Sprintf(`assert artifact --fingerprint 8e568bd886069f1290def0caabc1e97ce0e7b80c105e611258b57d76fcef234c --flow %s --api-token secret`, suite.flowName1),
- golden: "Error: --org is not set\nUsage: kosli assert artifact [IMAGE-NAME | FILE-PATH | DIR-PATH] [flags]\n",
+ wantError: true,
+ wantExitCode: 4,
+ name: "01 missing --org fails",
+ cmd: fmt.Sprintf(`assert artifact --fingerprint 8e568bd886069f1290def0caabc1e97ce0e7b80c105e611258b57d76fcef234c --flow %s --api-token secret`, suite.flowName1),
+ golden: "Error: --org is not set\nUsage: kosli assert artifact [IMAGE-NAME | FILE-PATH | DIR-PATH] [flags]\n",
  },
  {
  wantError: true,
@@ -95,9 +96,10 @@ func (suite *AssertArtifactCommandTestSuite) TestAssertArtifactCmd() {
  golden: "Error: Artifact with fingerprint '8e568bd886069f1290def0caabc1e97ce0e7b80c105e611258b57d76fcef234c' does not exist in flow 'assert-artifact-one' belonging to organization 'docs-cmd-test-user'\n",
  },
  {
- name: "03 asserting a single existing compliant artifact (using --fingerprint) results in OK and zero exit",
- cmd: fmt.Sprintf(`assert artifact --fingerprint %s %s`, suite.fingerprint1, suite.defaultKosliArguments),
- goldenRegex: "(?s)^COMPLIANT\n.*Attestation-name.*See more details at http://localhost(:8001)?/docs-cmd-test-user/flows/assert-artifact-one/artifacts/0089a849fce9c7c9128cd13a2e8b1c0757bdb6a7bad0fdf2800e38c19055b7fc(?:\\?artifact_id=[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{8})?\n",
+ name: "03 asserting a single existing compliant artifact (using --fingerprint) results in OK and zero exit",
+ wantExitCode: 0,
+ cmd: fmt.Sprintf(`assert artifact --fingerprint %s %s`, suite.fingerprint1, suite.defaultKosliArguments),
+ goldenRegex: "(?s)^COMPLIANT\n.*Attestation-name.*See more details at http://localhost(:8001)?/docs-cmd-test-user/flows/assert-artifact-one/artifacts/0089a849fce9c7c9128cd13a2e8b1c0757bdb6a7bad0fdf2800e38c19055b7fc(?:\\?artifact_id=[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{8})?\n",
  },
  {
  name: "04 json output of asserting a single existing compliant artifact (using --fingerprint) results in OK and zero exit",
@@ -176,28 +178,32 @@ func (suite *AssertArtifactCommandTestSuite) TestAssertArtifactCmd() {
  },
  },
  {
- wantError: true,
- name: "14 not providing --fingerprint nor --artifact-type fails",
- cmd: fmt.Sprintf(`assert artifact --flow %s %s`, suite.flowName1, suite.defaultKosliArguments),
- golden: "Error: docker image name or file/dir path is required when --fingerprint is not provided\nUsage: kosli assert artifact [IMAGE-NAME | FILE-PATH | DIR-PATH] [flags]\n",
+ wantError: true,
+ wantExitCode: 4,
+ name: "14 not providing --fingerprint nor --artifact-type fails",
+ cmd: fmt.Sprintf(`assert artifact --flow %s %s`, suite.flowName1, suite.defaultKosliArguments),
+ golden: "Error: docker image name or file/dir path is required when --fingerprint is not provided\nUsage: kosli assert artifact [IMAGE-NAME | FILE-PATH | DIR-PATH] [flags]\n",
  },
  {
- wantError: true,
- name: "15 providing both --environment and --polices fails",
- cmd: fmt.Sprintf(`assert artifact --fingerprint %s --environment %s --policy %s %s`, suite.fingerprint1, suite.envName, suite.policyName1, suite.defaultKosliArguments),
- golden: "Error: Cannot specify both 'environment_name' and 'policy_name' at the same time\n",
+ wantError: true,
+ wantExitCode: 1,
+ name: "15 providing both --environment and --polices fails",
+ cmd: fmt.Sprintf(`assert artifact --fingerprint %s --environment %s --policy %s %s`, suite.fingerprint1, suite.envName, suite.policyName1, suite.defaultKosliArguments),
+ golden: "Error: Cannot specify both 'environment_name' and 'policy_name' at the same time\n",
  },
  {
- wantError: true,
- name: "16 asserting a single existing non-compliant artifact (using --fingerprint) results in non-zero exit",
- cmd: fmt.Sprintf(`assert artifact --fingerprint %s %s`, suite.fingerprint3, suite.defaultKosliArguments),
- goldenRegex: "^Error: NON-COMPLIANT\n",
+ wantError: true,
+ wantExitCode: 1,
+ name: "16 asserting a single existing non-compliant artifact (using --fingerprint) results in non-zero exit",
+ cmd: fmt.Sprintf(`assert artifact --fingerprint %s %s`, suite.fingerprint3, suite.defaultKosliArguments),
+ goldenRegex: "^Error: NON-COMPLIANT\n",
  },
  {
- wantError: true,
- name: "17 asserting a single existing non-compliant artifact (using --artifact-type) results in non-zero exit",
- cmd: fmt.Sprintf(`assert artifact %s --artifact-type file %s`, suite.artifact3Path, suite.defaultKosliArguments),
- goldenRegex: "^Error: NON-COMPLIANT\n",
+ wantError: true,
+ wantExitCode: 1,
+ name: "17 asserting a single existing non-compliant artifact (using --artifact-type) results in non-zero exit",
+ cmd: fmt.Sprintf(`assert artifact %s --artifact-type file %s`, suite.artifact3Path, suite.defaultKosliArguments),
+ goldenRegex: "^Error: NON-COMPLIANT\n",
  },
 	}
 

cmd/kosli/assertPRAzure.go

@@ -5,6 +5,7 @@ import (
 	"io"
 
 	azUtils "github.com/kosli-dev/cli/internal/azure"
+	kosliErrors "github.com/kosli-dev/cli/internal/errors"
 	"github.com/spf13/cobra"
 )
 
@@ -64,7 +65,7 @@ func (o *assertPullRequestAzureOptions) run(args []string) error {
  return err
 	}
 	if len(pullRequestsEvidence) == 0 {
- return fmt.Errorf("assert failed: found no pull request(s) in Azure DevOps for commit: %s", o.commit)
+ return kosliErrors.NewErrCompliance(fmt.Sprintf("assert failed: found no pull request(s) in Azure DevOps for commit: %s", o.commit))
 	}
 	logger.Info("found [%d] pull request(s) in Azure DevOps for commit: %s", len(pullRequestsEvidence), o.commit)
 	return nil

cmd/kosli/assertPRAzure_test.go

@@ -30,22 +30,25 @@ func (suite *AssertPRAzureCommandTestSuite) SetupTest() {
 func (suite *AssertPRAzureCommandTestSuite) TestAssertPRAzureCmd() {
 	tests := []cmdTestCase{
  {
- name: "assert Azure PR evidence passes when commit has a PR in Azure",
- cmd: `assert pullrequest azure --azure-org-url https://dev.azure.com/kosli --project kosli-azure --repository cli 
+ name: "assert Azure PR evidence passes when commit has a PR in Azure",
+ wantExitCode: 0,
+ cmd: `assert pullrequest azure --azure-org-url https://dev.azure.com/kosli --project kosli-azure --repository cli
  --commit e6b38318747f1c225e6d2cdba1e88aa00fbcae29` + suite.defaultKosliArguments,
  golden: "found [1] pull request(s) in Azure DevOps for commit: e6b38318747f1c225e6d2cdba1e88aa00fbcae29\n",
  },
  {
- wantError: true,
- name: "assert Azure PR evidence fails when commit has no PRs in Azure",
- cmd: `assert pullrequest azure --azure-org-url https://dev.azure.com/kosli --project kosli-azure --repository cli 
+ wantError: true,
+ wantExitCode: 1,
+ name: "assert Azure PR evidence fails when commit has no PRs in Azure",
+ cmd: `assert pullrequest azure --azure-org-url https://dev.azure.com/kosli --project kosli-azure --repository cli
  --commit 58d8aad96e0dcd11ada3dc6650d23909eed336ed` + suite.defaultKosliArguments,
  golden: "Error: assert failed: found no pull request(s) in Azure DevOps for commit: 58d8aad96e0dcd11ada3dc6650d23909eed336ed\n",
  },
  {
- wantError: true,
- name: "assert Azure PR evidence fails when commit does not exist",
- cmd: `assert pullrequest azure --azure-org-url https://dev.azure.com/kosli --project kosli-azure --repository cli 
+ wantError: true,
+ wantExitCode: 1,
+ name: "assert Azure PR evidence fails when commit does not exist",
+ cmd: `assert pullrequest azure --azure-org-url https://dev.azure.com/kosli --project kosli-azure --repository cli
  --commit c4fa4c2ce6bef984abc93be9258a85f9137ff1c9` + suite.defaultKosliArguments,
  golden: "Error: assert failed: found no pull request(s) in Azure DevOps for commit: c4fa4c2ce6bef984abc93be9258a85f9137ff1c9\n",
  },

cmd/kosli/assertPRBitbucket.go

@@ -5,6 +5,7 @@ import (
 	"io"
 
 	bbUtils "github.com/kosli-dev/cli/internal/bitbucket"
+	kosliErrors "github.com/kosli-dev/cli/internal/errors"
 	"github.com/spf13/cobra"
 )
 
@@ -86,7 +87,7 @@ func (o *assertPullRequestBitbucketOptions) run(args []string) error {
  return err
 	}
 	if len(pullRequestsEvidence) == 0 {
- return fmt.Errorf("assert failed: found no pull request(s) in Bitbucket for commit: %s", o.commit)
+ return kosliErrors.NewErrCompliance(fmt.Sprintf("assert failed: found no pull request(s) in Bitbucket for commit: %s", o.commit))
 	}
 	logger.Info("found [%d] pull request(s) in Bitbucket for commit: %s", len(pullRequestsEvidence), o.commit)
 	return nil