From 0ae00ab65713d90ca6684b41a3800f7ecb420a63 Mon Sep 17 00:00:00 2001 From: John Ragsdale Date: Wed, 18 Sep 2019 15:27:05 -0700 Subject: [PATCH 01/11] updated path to set_secret.py in deploy_steps.yml --- .ci/steps/deploy_steps.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/steps/deploy_steps.yml b/.ci/steps/deploy_steps.yml index e3cc93e..a052dc0 100644 --- a/.ci/steps/deploy_steps.yml +++ b/.ci/steps/deploy_steps.yml @@ -53,7 +53,7 @@ steps: inlineScript: | source activate ${{parameters.conda}} pip install azure azure-cli - python .ci/scripts/set_secret.py -n "${{parameters.ENVIRONMENT_PREFIX}}-key" + python ../../.ci/scripts/set_secret.py -n "${{parameters.ENVIRONMENT_PREFIX}}-key" - task: AzurePowerShell@4 From e0ddbab36707dd3142180d747e3afcee1f288723 Mon Sep 17 00:00:00 2001 From: John Ragsdale Date: Wed, 18 Sep 2019 16:54:25 -0700 Subject: [PATCH 02/11] updating set_secret.py path --- .ci/steps/deploy_steps.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/steps/deploy_steps.yml b/.ci/steps/deploy_steps.yml index a052dc0..b0840bc 100644 --- a/.ci/steps/deploy_steps.yml +++ b/.ci/steps/deploy_steps.yml @@ -53,7 +53,7 @@ steps: inlineScript: | source activate ${{parameters.conda}} pip install azure azure-cli - python ../../.ci/scripts/set_secret.py -n "${{parameters.ENVIRONMENT_PREFIX}}-key" + python AIArchitecturesAndPractices/.ci/scripts/set_secret.py -n "${{parameters.ENVIRONMENT_PREFIX}}-key" - task: AzurePowerShell@4 From 29ace8e4110c9171b626d5e124fdadb69007a3f9 Mon Sep 17 00:00:00 2001 From: John Ragsdale Date: Wed, 18 Sep 2019 19:18:40 -0700 Subject: [PATCH 03/11] not using AzureRM this time --- .ci/steps/deploy_steps.yml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.ci/steps/deploy_steps.yml b/.ci/steps/deploy_steps.yml index b0840bc..fce3cf5 100644 --- a/.ci/steps/deploy_steps.yml +++ b/.ci/steps/deploy_steps.yml @@ -62,8 +62,7 @@ steps: ScriptType: 'InlineScript' FailOnStandardError: true azurePowerShellVersion: 'LatestVersion' - Inline: | - Install-Module -Name AzureRM.Resources -RequiredVersion 1.0.3 + Inline: | $resourceGroupName = "${{parameters.azureresourcegroup}}" $projectTags = New-Object 'System.Collections.Generic.Dictionary[String,String]' @@ -73,7 +72,7 @@ steps: { param( [string]$resGroup, $tags ) Write-Host("Updating resources in " + $resGroup) - $resources = Get-AzureRmResource -ResourceGroupName $resGroup + $resources = Get-AzResource -ResourceGroupName $resGroup $resources | ForEach-Object { if($_.Tags -eq $null) @@ -90,13 +89,13 @@ steps: } $resources | ForEach-Object { - Set-AzureRmResource -ResourceId $_.Id -Tag $_.Tags -Force + Set-AzResource -ResourceId $_.Id -Tag $_.Tags -Force } } Update-GroupResources -resGroup $resourceGroupName -tags $projectTags - $clusterResources = Get-AzureRmResource -ResourceType "Microsoft.ContainerService/managedClusters" -ResourceGroupName $resourceGroupName -ExpandProperties + $clusterResources = Get-AzResource -ResourceType "Microsoft.ContainerService/managedClusters" -ResourceGroupName $resourceGroupName -ExpandProperties foreach($cluster in $clusterResources) { Update-GroupResources -resGroup $cluster.Properties.nodeResourceGroup -tags $projectTags From 11f1703f02175f826283508169a6a2f38db3fdd7 Mon Sep 17 00:00:00 2001 From: John Ragsdale Date: Wed, 18 Sep 2019 20:34:45 -0700 Subject: [PATCH 04/11] adding exception info to the python script --- .ci/scripts/set_secret.py | 4 ++-- .ci/steps/deploy_steps.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.ci/scripts/set_secret.py b/.ci/scripts/set_secret.py index 057e83f..0ef885d 100644 --- a/.ci/scripts/set_secret.py +++ b/.ci/scripts/set_secret.py @@ -13,8 +13,8 @@ def set_secret(kv_endpoint, secret_name, secret_value): return "Successfully created secret: {secret_name} in keyvault: {kv_endpoint}".format( secret_name=secret_name, kv_endpoint=kv_endpoint) except Exception as ex: - return "Failed to create secret: {secret_name} in keyvault: {kv_endpoint}".format( - secret_name=secret_name, kv_endpoint=kv_endpoint) + return "Failed to create secret: {secret_name} in keyvault: {kv_endpoint} because exception: {kv_exception}".format( + secret_name=secret_name, kv_endpoint=kv_endpoint, kv_exception=ex) def parse_args(): diff --git a/.ci/steps/deploy_steps.yml b/.ci/steps/deploy_steps.yml index fce3cf5..29cae39 100644 --- a/.ci/steps/deploy_steps.yml +++ b/.ci/steps/deploy_steps.yml @@ -61,7 +61,7 @@ steps: azureSubscription: ${{parameters.azureSubscription}} ScriptType: 'InlineScript' FailOnStandardError: true - azurePowerShellVersion: 'LatestVersion' + azurePowerShellVersion: 'latestVersion' Inline: | $resourceGroupName = "${{parameters.azureresourcegroup}}" From 21033ba4a96cdadd7e47ae09a2078c06132dee5c Mon Sep 17 00:00:00 2001 From: John Ragsdale Date: Wed, 18 Sep 2019 20:38:26 -0700 Subject: [PATCH 05/11] removing try catch --- .ci/scripts/set_secret.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/.ci/scripts/set_secret.py b/.ci/scripts/set_secret.py index 0ef885d..1177c16 100644 --- a/.ci/scripts/set_secret.py +++ b/.ci/scripts/set_secret.py @@ -1,5 +1,4 @@ #!/usr/bin/python3 - import argparse from azure.keyvault import KeyVaultClient from azure.common.client_factory import get_client_from_cli_profile @@ -8,13 +7,8 @@ def set_secret(kv_endpoint, secret_name, secret_value): client = get_client_from_cli_profile(KeyVaultClient) - try: - client.set_secret(kv_endpoint, secret_name, secret_value) - return "Successfully created secret: {secret_name} in keyvault: {kv_endpoint}".format( - secret_name=secret_name, kv_endpoint=kv_endpoint) - except Exception as ex: - return "Failed to create secret: {secret_name} in keyvault: {kv_endpoint} because exception: {kv_exception}".format( - secret_name=secret_name, kv_endpoint=kv_endpoint, kv_exception=ex) + client.set_secret(kv_endpoint, secret_name, secret_value) + return "Successfully created secret: {secret_name} in keyvault: {kv_endpoint}".format(secret_name=secret_name, kv_endpoint=kv_endpoint) def parse_args(): From f2882f6993a13dc60e67714098a5b37aa6e20125 Mon Sep 17 00:00:00 2001 From: John Ragsdale Date: Wed, 18 Sep 2019 20:45:17 -0700 Subject: [PATCH 06/11] adding deployment-id to resource group --- .ci/scripts/set_secret.py | 3 +-- .ci/steps/createResourceGroupTemplate.yml | 4 +++- .ci/steps/deploy_steps.yml | 1 + 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.ci/scripts/set_secret.py b/.ci/scripts/set_secret.py index 1177c16..721ffd4 100644 --- a/.ci/scripts/set_secret.py +++ b/.ci/scripts/set_secret.py @@ -14,8 +14,7 @@ def set_secret(kv_endpoint, secret_name, secret_value): def parse_args(): parser = argparse.ArgumentParser() - parser.add_argument('-n', '--secretName', required=True, - help="The name of the secret") + parser.add_argument('-n', '--secretName', required=True, help="The name of the secret") return parser.parse_args() diff --git a/.ci/steps/createResourceGroupTemplate.yml b/.ci/steps/createResourceGroupTemplate.yml index 65988db..ab475c8 100644 --- a/.ci/steps/createResourceGroupTemplate.yml +++ b/.ci/steps/createResourceGroupTemplate.yml @@ -5,6 +5,8 @@ parameters: alias : $(Build.RequestedForId) project : "e2etestharness" expires : "2019-08-01" + deploymentguidtag: "ABC123 + steps: - task: AzureCLI@1 @@ -14,4 +16,4 @@ steps: scriptLocation: inlineScript inlineScript: | echo Execute Resource Group Delete - az group create -l ${{parameters.location}} -n ${{parameters.azureresourcegroup}} --tags alias='${{parameters.alias}}' project='${{parameters.project}}' expires='${{parameters.expires}}' + az group create -l ${{parameters.location}} -n ${{parameters.azureresourcegroup}} --tags deployment-id='${{parameters.deploymentguidtag}}' alias='${{parameters.alias}}' project='${{parameters.project}}' expires='${{parameters.expires}}' diff --git a/.ci/steps/deploy_steps.yml b/.ci/steps/deploy_steps.yml index 29cae39..0d84d5b 100644 --- a/.ci/steps/deploy_steps.yml +++ b/.ci/steps/deploy_steps.yml @@ -32,6 +32,7 @@ steps: alias : ${{parameters.alias}} project : ${{parameters.project}} expires : ${{parameters.expires}} + deploymentguidtag: ${{parameters.deploymentguidtag}} - template: ${{parameters.template}} parameters: From 592ff63172ec076f4000c5cd4c959faa875cfdc7 Mon Sep 17 00:00:00 2001 From: John Ragsdale Date: Wed, 18 Sep 2019 20:46:31 -0700 Subject: [PATCH 07/11] yaml formatting --- .ci/steps/createResourceGroupTemplate.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/steps/createResourceGroupTemplate.yml b/.ci/steps/createResourceGroupTemplate.yml index ab475c8..4f26fa1 100644 --- a/.ci/steps/createResourceGroupTemplate.yml +++ b/.ci/steps/createResourceGroupTemplate.yml @@ -5,7 +5,7 @@ parameters: alias : $(Build.RequestedForId) project : "e2etestharness" expires : "2019-08-01" - deploymentguidtag: "ABC123 + deploymentguidtag: "ABC123" steps: From 7bc5f430a1eb0bb76cd5ae792d64574df34c3ab0 Mon Sep 17 00:00:00 2001 From: John Ragsdale Date: Wed, 18 Sep 2019 22:09:11 -0700 Subject: [PATCH 08/11] trying to store secret with powershell instead of python --- .ci/steps/deploy_steps.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.ci/steps/deploy_steps.yml b/.ci/steps/deploy_steps.yml index 0d84d5b..479c34a 100644 --- a/.ci/steps/deploy_steps.yml +++ b/.ci/steps/deploy_steps.yml @@ -64,6 +64,10 @@ steps: FailOnStandardError: true azurePowerShellVersion: 'latestVersion' Inline: | + + $secretvalue = ConvertTo-SecureString -String "$env:storage_conn_string" -AsPlainText -Force + Set-AzKeyVaultSecret -VaultName 't3scriptkeyvault' -Name "${{parameters.ENVIRONMENT_PREFIX}}-key" -SecretValue $secretvalue + $resourceGroupName = "${{parameters.azureresourcegroup}}" $projectTags = New-Object 'System.Collections.Generic.Dictionary[String,String]' From 44b7284d882bad7ad14db197ad1109dfea0c214a Mon Sep 17 00:00:00 2001 From: John Ragsdale Date: Thu, 19 Sep 2019 00:55:09 -0700 Subject: [PATCH 09/11] removed keyvault from powershell --- .ci/steps/deploy_steps.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.ci/steps/deploy_steps.yml b/.ci/steps/deploy_steps.yml index 479c34a..517f416 100644 --- a/.ci/steps/deploy_steps.yml +++ b/.ci/steps/deploy_steps.yml @@ -65,9 +65,6 @@ steps: azurePowerShellVersion: 'latestVersion' Inline: | - $secretvalue = ConvertTo-SecureString -String "$env:storage_conn_string" -AsPlainText -Force - Set-AzKeyVaultSecret -VaultName 't3scriptkeyvault' -Name "${{parameters.ENVIRONMENT_PREFIX}}-key" -SecretValue $secretvalue - $resourceGroupName = "${{parameters.azureresourcegroup}}" $projectTags = New-Object 'System.Collections.Generic.Dictionary[String,String]' From fbd52473ab28e89bb1f3b4f09fb1452c3d292e5e Mon Sep 17 00:00:00 2001 From: John Ragsdale Date: Thu, 19 Sep 2019 11:28:40 -0700 Subject: [PATCH 10/11] adding path correctio to set_secret.py python script location --- .ci/steps/deploy_steps.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/steps/deploy_steps.yml b/.ci/steps/deploy_steps.yml index a5af773..b929791 100644 --- a/.ci/steps/deploy_steps.yml +++ b/.ci/steps/deploy_steps.yml @@ -53,7 +53,7 @@ steps: inlineScript: | source activate ${{parameters.conda}} pip install azure azure-cli - python .ci/scripts/set_secret.py -n "${{parameters.ENVIRONMENT_PREFIX}}-key" + python AIArchitecturesAndPractices/.ci/scripts/set_secret.py -n "${{parameters.ENVIRONMENT_PREFIX}}-key" - task: AzurePowerShell@4 From 5ae64ed59657965d9a9bbf38c3aaf355ccb05a48 Mon Sep 17 00:00:00 2001 From: John Ragsdale Date: Thu, 19 Sep 2019 14:38:55 -0700 Subject: [PATCH 11/11] plubing in deployment id for resource group --- .ci/steps/deploy_steps.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.ci/steps/deploy_steps.yml b/.ci/steps/deploy_steps.yml index b929791..fe08970 100644 --- a/.ci/steps/deploy_steps.yml +++ b/.ci/steps/deploy_steps.yml @@ -32,6 +32,7 @@ steps: alias : ${{parameters.alias}} project : ${{parameters.project}} expires : ${{parameters.expires}} + deploymentguidtag: ${{parameters.deploymentguidtag}} - template: ${{parameters.template}} parameters: