Represent struct fields and map keys; fix ss_default to ignore map keys

acceptance/bin/edit_resource.py

@@ -0,0 +1,62 @@
+#!/usr/bin/env python3
+"""
+Implements get / update / set for a given resource.
+
+Fetches a given resource by id from the backend, executes Python code read from stdin and updates the resource.
+"""
+
+import sys
+import os
+import subprocess
+import argparse
+import json
+import pprint
+
+sys.path.insert(0, os.path.dirname(__file__))
+import util
+from util import run_json, run
+
+
+CLI = os.environ["CLI"]
+
+
+# Each class should be named after CLI command group and implement get(id) and set(id, value) methods:
+
+
+class jobs:
+ def get(self, job_id):
+ return run_json([CLI, "jobs", "get", job_id])["settings"]
+
+ def set(self, job_id, value):
+ payload = {"job_id": job_id, "new_settings": value}
+ return run([CLI, "jobs", "reset", job_id, "--json", json.dumps(payload)])
+
+
+def main():
+ parser = argparse.ArgumentParser()
+ parser.add_argument("type")
+ parser.add_argument("id")
+ parser.add_argument("-v", "--verbose", action="store_true")
+ args = parser.parse_args()
+
+ util.VERBOSE = args.verbose
+
+ script = sys.stdin.read()
+
+ klass = globals()[args.type]
+ instance = klass()
+
+ data = instance.get(args.id)
+ my_locals = {"r": data}
+
+ try:
+ exec(script, locals=my_locals)
+ except Exception:
+ pprint.pprint(my_locals)
+ raise
+
+ instance.set(args.id, my_locals["r"])
+
+
+if __name__ == "__main__":
+ main()

acceptance/bin/util.py

@@ -0,0 +1,34 @@
+import sys
+import subprocess
+import json
+import shlex
+
+
+VERBOSE = False
+
+
+class RunError(Exception):
+ pass
+
+
+def run_json(cmd):
+ if VERBOSE:
+ print("+ " + " ".join([shlex.quote(x) for x in cmd]), file=sys.stderr, flush=True)
+ result = subprocess.run(cmd, stdout=subprocess.PIPE, encoding="utf-8")
+ if VERBOSE and result.stdout:
+ print(result.stdout, flush=True)
+ if result.returncode != 0:
+ raise RunError(f"{cmd} failed with code {result.returncode}\n{result.stdout}".strip())
+ try:
+ return json.loads(result.stdout)
+ except Exception as ex:
+ raise RunError(f"{cmd} returned non-json: {ex}\n{result.stdout}")
+
+
+def run(cmd):
+ if VERBOSE:
+ print("+ " + " ".join([shlex.quote(x) for x in cmd]), file=sys.stderr, flush=True)
+ result = subprocess.run(cmd)
+ if result.returncode != 0:
+ raise RunError(f"{cmd} failed with code {result.returncode}")
+ return result

acceptance/bundle/migrate/basic/out.plan_update.json

@@ -175,7 +175,7 @@
  "name": {
  "action": "update"
  },
- "tags.myjob_name": {
+ "tags['myjob_name']": {
  "action": "update"
  }
  },

acceptance/bundle/resources/jobs/remote_add_tag/databricks.yml

@@ -0,0 +1,14 @@
+resources:
+ jobs:
+ foo:
+ name: "My Wheel Job"
+ tags:
+ tag1: tag value
+ tasks:
+ - task_key: TestTask
+ python_wheel_task:
+ package_name: "my_test_code"
+ entry_point: "run"
+ environment_key: test_env
+ libraries:
+ - whl: hello.whl

acceptance/bundle/resources/jobs/remote_add_tag/out.plan_post_update.direct.json

@@ -0,0 +1,100 @@
+{
+ "plan": {
+ "resources.jobs.foo": {
+ "action": "update",
+ "new_state": {
+ "value": {
+ "deployment": {
+ "kind": "BUNDLE",
+ "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json"
+ },
+ "edit_mode": "UI_LOCKED",
+ "format": "MULTI_TASK",
+ "max_concurrent_runs": 1,
+ "name": "My Wheel Job",
+ "queue": {
+ "enabled": true
+ },
+ "tags": {
+ "tag1": "tag value"
+ },
+ "tasks": [
+ {
+ "environment_key": "test_env",
+ "libraries": [
+ {
+ "whl": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/hello.whl"
+ }
+ ],
+ "python_wheel_task": {
+ "entry_point": "run",
+ "package_name": "my_test_code"
+ },
+ "task_key": "TestTask"
+ }
+ ]
+ }
+ },
+ "remote_state": {
+ "created_time": [UNIX_TIME_MILLIS],
+ "creator_user_name": "[USERNAME]",
+ "job_id": [JOB_ID],
+ "run_as_user_name": "[USERNAME]",
+ "settings": {
+ "deployment": {
+ "kind": "BUNDLE",
+ "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json"
+ },
+ "edit_mode": "UI_LOCKED",
+ "email_notifications": {},
+ "format": "MULTI_TASK",
+ "max_concurrent_runs": 1,
+ "name": "My Wheel Job",
+ "queue": {
+ "enabled": true
+ },
+ "tags": {
+ "new_tag": "new_value",
+ "tag1": "tag value"
+ },
+ "tasks": [
+ {
+ "environment_key": "test_env",
+ "libraries": [
+ {
+ "whl": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/hello.whl"
+ }
+ ],
+ "python_wheel_task": {
+ "entry_point": "run",
+ "package_name": "my_test_code"
+ },
+ "task_key": "TestTask"
+ }
+ ],
+ "timeout_seconds": 0,
+ "webhook_notifications": {}
+ }
+ },
+ "changes": {
+ "remote": {
+ "email_notifications": {
+ "action": "skip",
+ "reason": "server_side_default"
+ },
+ "tags['new_tag']": {
+ "action": "update"
+ },
+ "timeout_seconds": {
+ "action": "skip",
+ "reason": "server_side_default"
+ },
+ "webhook_notifications": {
+ "action": "skip",
+ "reason": "server_side_default"
+ }
+ }
+ }
+ }
+ }
+}

acceptance/bundle/resources/jobs/remote_add_tag/out.plan_post_update.terraform.json

@@ -0,0 +1,7 @@
+{
+ "plan": {
+ "resources.jobs.foo": {
+ "action": "update"
+ }
+ }
+}

acceptance/bundle/resources/jobs/remote_add_tag/output.txt

@@ -0,0 +1,11 @@
+create jobs.foo
+
+Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged
+Uploading hello.whl...
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+update jobs.foo
+
+Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged

acceptance/bundle/resources/jobs/remote_add_tag/script

@@ -0,0 +1,12 @@
+touch hello.whl
+$CLI bundle plan
+$CLI bundle deploy
+job_id="$(read_id.py jobs foo)"
+echo "$job_id:JOB_ID" >> ACC_REPLS
+
+edit_resource.py jobs $job_id <<EOF
+r["tags"]["new_tag"] = "new_value"
+EOF
+
+$CLI bundle plan
+$CLI bundle plan -o json > out.plan_post_update.$DATABRICKS_BUNDLE_ENGINE.json

acceptance/bundle/resources/jobs/remote_add_tag/test.toml

@@ -0,0 +1,2 @@
+RecordRequests = false
+Ignore = [".databricks", "hello.whl"]

bundle/direct/bundle_plan.go

@@ -260,12 +260,11 @@ func interpretOldStateVsRemoteState(ctx context.Context, adapter *dresources.Ada
 	m := make(map[string]deployplan.Trigger)
 
 	for _, ch := range diff {
- if ch.Old == nil && ch.Path.IsStringKey() {
+ if ch.Old == nil && ch.Path.IsDotString() {
  // The field was not set by us, but comes from the remote state.
  // This could either be server-side default or a policy.
  // In any case, this is not a change we should react to.
- // Note, we only consider StringKeys here, because indexes and key-value pairs refer to slices and we want to react to new element in slices.
- // Note, IsStringKey is also too broad - it currently covers struct fields and map keys, we don't want to include map keys here.
+ // Note, we only consider struct fields here. Adding/removing elements to/from maps and slices should trigger updates.
  m[ch.Path.String()] = deployplan.Trigger{
  Action: deployplan.ActionTypeSkipString,
  Reason: "server_side_default",

libs/structs/structdiff/diff.go

@@ -205,7 +205,7 @@ func diffStruct(ctx *diffContext, path *structpath.PathNode, s1, s2 reflect.Valu
  if fieldName == "" {
  fieldName = sf.Name
  }
- node := structpath.NewStringKey(path, fieldName)
+ node := structpath.NewDotString(path, fieldName)
 
  v1Field := s1.Field(i)
  v2Field := s2.Field(i)
@@ -257,7 +257,7 @@ func diffMapStringKey(ctx *diffContext, path *structpath.PathNode, m1, m2 reflec
  k := keySet[ks]
  v1 := m1.MapIndex(k)
  v2 := m2.MapIndex(k)
- node := structpath.NewStringKey(path, ks)
+ node := structpath.NewBracketString(path, ks)
  if err := diffValues(ctx, node, v1, v2, changes); err != nil {
  return err
  }

libs/structs/structdiff/diff_test.go

@@ -147,7 +147,7 @@ func TestGetStructDiff(t *testing.T) {
  name: "map diff",
  a: A{M: map[string]int{"a": 1}},
  b: A{M: map[string]int{"a": 2}},
- want: []ResolvedChange{{Field: "m.a", Old: 1, New: 2}},
+ want: []ResolvedChange{{Field: "m['a']", Old: 1, New: 2}},
  },
  {
  name: "slice diff",
@@ -258,9 +258,9 @@ func TestGetStructDiff(t *testing.T) {
  a: map[string]C{"key1": {Title: "title", ForceSendFields: []string{"Name", "IsEnabled", "Title"}}},
  b: map[string]C{"key1": {Title: "title", ForceSendFields: []string{"Age"}}},
  want: []ResolvedChange{
- {Field: "key1.name", Old: "", New: nil},
- {Field: "key1.age", Old: nil, New: 0},
- {Field: "key1.is_enabled", Old: false, New: nil},
+ {Field: "['key1'].name", Old: "", New: nil},
+ {Field: "['key1'].age", Old: nil, New: 0},
+ {Field: "['key1'].is_enabled", Old: false, New: nil},
  },
  },
 
@@ -270,9 +270,9 @@ func TestGetStructDiff(t *testing.T) {
  a: map[string]*C{"key1": {Title: "title", ForceSendFields: []string{"Name", "IsEnabled", "Title"}}},
  b: map[string]*C{"key1": {Title: "title", ForceSendFields: []string{"Age"}}},
  want: []ResolvedChange{
- {Field: "key1.name", Old: "", New: nil},
- {Field: "key1.age", Old: nil, New: 0},
- {Field: "key1.is_enabled", Old: false, New: nil},
+ {Field: "['key1'].name", Old: "", New: nil},
+ {Field: "['key1'].age", Old: nil, New: 0},
+ {Field: "['key1'].is_enabled", Old: false, New: nil},
  },
  },