Coverage for acceptance tests

.gitignore

@@ -20,6 +20,7 @@ dist/
 
 *.log
 coverage.txt
+coverage-acceptance.txt
 
 __pycache__
 *.pyc

Makefile

@@ -25,6 +25,17 @@ cover:
 showcover:
 	go tool cover -html=coverage.txt
 
+acc-cover:
+	rm -fr ./acceptance/build/cover/
+	CLI_GOCOVERDIR=build/cover go test ./acceptance
+	rm -fr ./acceptance/build/cover-merged/
+	mkdir -p acceptance/build/cover-merged/
+	go tool covdata merge -i $$(printf '%s,' acceptance/build/cover/* | sed 's/,$$//') -o acceptance/build/cover-merged/
+	go tool covdata textfmt -i acceptance/build/cover-merged -o coverage-acceptance.txt
+
+acc-showcover:
+	go tool cover -html=coverage-acceptance.txt
+
 build: vendor
 	go build -mod vendor
 
@@ -45,4 +56,4 @@ integration:
 integration-short:
 	$(INTEGRATION) -short
 
-.PHONY: lint lintcheck fmt test cover showcover build snapshot vendor schema integration integration-short
+.PHONY: lint lintcheck fmt test cover showcover build snapshot vendor schema integration integration-short acc-cover acc-showcover

acceptance/acceptance_test.go

@@ -38,7 +38,16 @@ func TestAccept(t *testing.T) {
 	cwd, err := os.Getwd()
 	require.NoError(t, err)
 
-	execPath := BuildCLI(t, cwd)
+	coverDir := os.Getenv("CLI_GOCOVERDIR")
+
+	if coverDir != "" {
+ require.NoError(t, os.MkdirAll(coverDir, os.ModePerm))
+ coverDir, err = filepath.Abs(coverDir)
+ require.NoError(t, err)
+ t.Logf("Writing coverage to %s", coverDir)
+	}
+
+	execPath := BuildCLI(t, cwd, coverDir)
 	// $CLI is what test scripts are using
 	t.Setenv("CLI", execPath)
 
@@ -60,10 +69,11 @@ func TestAccept(t *testing.T) {
 
 	testDirs := getTests(t)
 	require.NotEmpty(t, testDirs)
+
 	for _, dir := range testDirs {
  t.Run(dir, func(t *testing.T) {
  t.Parallel()
- runTest(t, dir, repls)
+ runTest(t, dir, coverDir, repls)
  })
 	}
 }
@@ -88,7 +98,7 @@ func getTests(t *testing.T) []string {
 	return testDirs
 }
 
-func runTest(t *testing.T, dir string, repls testdiff.ReplacementsContext) {
+func runTest(t *testing.T, dir, coverDir string, repls testdiff.ReplacementsContext) {
 	var tmpDir string
 	var err error
 	if KeepTmp {
@@ -111,6 +121,15 @@ func runTest(t *testing.T, dir string, repls testdiff.ReplacementsContext) {
 
 	args := []string{"bash", "-euo", "pipefail", EntryPointScript}
 	cmd := exec.Command(args[0], args[1:]...)
+	if coverDir != "" {
+ // Creating individual coverage directory for each test, because writing to the same one
+ // results in sporadic failures like this one (only if tests are running in parallel):
+ // +error: coverage meta-data emit failed: writing ... rename .../tmp.covmeta.b3f... .../covmeta.b3f2c...: no such file or directory
+ coverDir = filepath.Join(coverDir, strings.ReplaceAll(dir, string(os.PathSeparator), "--"))
+ err := os.MkdirAll(coverDir, os.ModePerm)
+ require.NoError(t, err)
+ cmd.Env = append(os.Environ(), "GOCOVERDIR="+coverDir)
+	}
 	cmd.Dir = tmpDir
 	outB, err := cmd.CombinedOutput()
 
@@ -210,14 +229,17 @@ func readMergedScriptContents(t *testing.T, dir string) string {
 	return strings.Join(prepares, "\n")
 }
 
-func BuildCLI(t *testing.T, cwd string) string {
+func BuildCLI(t *testing.T, cwd, coverDir string) string {
 	execPath := filepath.Join(cwd, "build", "databricks")
 	if runtime.GOOS == "windows" {
  execPath += ".exe"
 	}
 
 	start := time.Now()
 	args := []string{"go", "build", "-mod", "vendor", "-o", execPath}
+	if coverDir != "" {
+ args = append(args, "-cover")
+	}
 	cmd := exec.Command(args[0], args[1:]...)
 	cmd.Dir = ".."
 	out, err := cmd.CombinedOutput()