diff --git a/acceptance/bundle/resources/model_serving_endpoints/drift/recreated_same_name/databricks.yml.tmpl b/acceptance/bundle/resources/model_serving_endpoints/drift/recreated_same_name/databricks.yml.tmpl new file mode 100644 index 00000000000..8dcc1302127 --- /dev/null +++ b/acceptance/bundle/resources/model_serving_endpoints/drift/recreated_same_name/databricks.yml.tmpl @@ -0,0 +1,13 @@ +bundle: + name: drift-mse-recreated-same-name-$UNIQUE_NAME + +sync: + paths: [] + +resources: + model_serving_endpoints: + my_endpoint: + name: mse-endpoint-$UNIQUE_NAME + permissions: + - level: CAN_VIEW + user_name: deco-test-user@databricks.com diff --git a/acceptance/bundle/resources/model_serving_endpoints/drift/recreated_same_name/out.test.toml b/acceptance/bundle/resources/model_serving_endpoints/drift/recreated_same_name/out.test.toml new file mode 100644 index 00000000000..fe4076cdf9b --- /dev/null +++ b/acceptance/bundle/resources/model_serving_endpoints/drift/recreated_same_name/out.test.toml @@ -0,0 +1,4 @@ +Local = true +Cloud = true +RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/model_serving_endpoints/drift/recreated_same_name/output.txt b/acceptance/bundle/resources/model_serving_endpoints/drift/recreated_same_name/output.txt new file mode 100644 index 00000000000..fea354c1f8a --- /dev/null +++ b/acceptance/bundle/resources/model_serving_endpoints/drift/recreated_same_name/output.txt @@ -0,0 +1,63 @@ + +=== Initial deployment +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/drift-mse-recreated-same-name-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> [CLI] serving-endpoints get mse-endpoint-[UNIQUE_NAME] +{ + "name": "mse-endpoint-[UNIQUE_NAME]", + "creator": "[USERNAME]" +} + +=== Delete and recreate remotely with the same name +>>> [CLI] serving-endpoints delete mse-endpoint-[UNIQUE_NAME] + +>>> [CLI] serving-endpoints create mse-endpoint-[UNIQUE_NAME] --no-wait +{ + "name": "mse-endpoint-[UNIQUE_NAME]", + "creator": "[USERNAME]" +} + +>>> [CLI] serving-endpoints get mse-endpoint-[UNIQUE_NAME] +{ + "name": "mse-endpoint-[UNIQUE_NAME]", + "creator": "[USERNAME]" +} +Original endpoint id: [ORIGINAL_ENDPOINT_ID] +Remote recreated endpoint id: [REMOTE_RECREATED_ENDPOINT_ID] + +=== Plan after out-of-band recreate +>>> [CLI] bundle plan +update model_serving_endpoints.my_endpoint.permissions + +Plan: 0 to add, 1 to change, 0 to delete, 1 unchanged + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/drift-mse-recreated-same-name-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> [CLI] serving-endpoints get mse-endpoint-[UNIQUE_NAME] +{ + "name": "mse-endpoint-[UNIQUE_NAME]", + "creator": "[USERNAME]" +} + +=== Persistent drift after deploy: V1 permissions API leaves an ACL on the deleted endpoint id, so plan keeps showing an update. +>>> [CLI] bundle plan +update model_serving_endpoints.my_endpoint.permissions + +Plan: 0 to add, 1 to change, 0 to delete, 1 unchanged + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.model_serving_endpoints.my_endpoint + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/drift-mse-recreated-same-name-[UNIQUE_NAME]/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/resources/model_serving_endpoints/drift/recreated_same_name/script b/acceptance/bundle/resources/model_serving_endpoints/drift/recreated_same_name/script new file mode 100644 index 00000000000..9f764d7914e --- /dev/null +++ b/acceptance/bundle/resources/model_serving_endpoints/drift/recreated_same_name/script @@ -0,0 +1,41 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + trace $CLI bundle destroy --auto-approve + rm -f out.requests.txt +} +trap cleanup EXIT + +endpoint_name="mse-endpoint-${UNIQUE_NAME}" + +title "Initial deployment" +trace $CLI bundle deploy + +original_endpoint_id=$($CLI serving-endpoints get "${endpoint_name}" | jq -r '.id') +add_repl.py "$original_endpoint_id" "ORIGINAL_ENDPOINT_ID" +trace $CLI serving-endpoints get "${endpoint_name}" | jq '{name, creator}' + +title "Delete and recreate remotely with the same name" +trace $CLI serving-endpoints delete "${endpoint_name}" +trace $CLI serving-endpoints create "${endpoint_name}" --no-wait | jq '{name, creator}' + +remote_recreated_endpoint_id=$($CLI serving-endpoints get "${endpoint_name}" | jq -r '.id') +add_repl.py "$remote_recreated_endpoint_id" "REMOTE_RECREATED_ENDPOINT_ID" +trace $CLI serving-endpoints get "${endpoint_name}" | jq '{name, creator}' + +printf "Original endpoint id: %s\n" "$original_endpoint_id" +printf "Remote recreated endpoint id: %s\n" "$remote_recreated_endpoint_id" + +if [ "$original_endpoint_id" = "$remote_recreated_endpoint_id" ]; then + echo "Expected remote recreation to assign a different endpoint id" >&2 + exit 1 +fi + +title "Plan after out-of-band recreate" +trace $CLI bundle plan + +trace $CLI bundle deploy +trace $CLI serving-endpoints get "${endpoint_name}" | jq '{name, creator}' + +title "Persistent drift after deploy: V1 permissions API leaves an ACL on the deleted endpoint id, so plan keeps showing an update." +trace $CLI bundle plan diff --git a/acceptance/bundle/resources/model_serving_endpoints/drift/recreated_same_name/test.toml b/acceptance/bundle/resources/model_serving_endpoints/drift/recreated_same_name/test.toml new file mode 100644 index 00000000000..83e36142b53 --- /dev/null +++ b/acceptance/bundle/resources/model_serving_endpoints/drift/recreated_same_name/test.toml @@ -0,0 +1,8 @@ +Badness = "After deleting and recreating a model serving endpoint remotely with the same name but a different endpoint_id, bundle plan/deploy ends up with a permanent update on permissions because the V1 permissions API does not delete ACLs immediately when the parent is gone." + +Local = true +Cloud = true +RequiresUnityCatalog = true +RecordRequests = false + +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"]