Iotdpstags

Dockerfile

@@ -36,7 +36,7 @@ LABEL maintainer="Microsoft" \
 # libintl and icu-libs - required by azure devops artifact (az extension add --name azure-devops)
 RUN apk add --no-cache bash openssh ca-certificates jq curl openssl perl git zip \
  && apk add --no-cache --virtual .build-deps gcc make openssl-dev libffi-dev musl-dev linux-headers \
- && apk add --no-cache libintl icu-libs libc6-compat \
+ && apk add --no-cache libintl icu-libs libc6-compat postgresql-dev \
  && apk add --no-cache bash-completion \
  && update-ca-certificates
 

azure-pipelines.yml

@@ -107,7 +107,7 @@ jobs:
   - task: UsePythonVersion@0
     displayName: 'Use Python 3'
     inputs:
-      versionSpec: 3.x
+      versionSpec: 3.8
 
   - bash: ./scripts/ci/dependency_check.sh
     displayName: 'Verify src/azure-cli/requirements.py3.Darwin.txt'

src/azure-cli-core/HISTORY.rst

@@ -3,6 +3,10 @@
 Release History
 ===============
 
+2.14.0
+++++++
+* Minor fixes
+
 2.13.0
 ++++++
 * Minor fixes

src/azure-cli-core/azure/cli/core/__init__.py

@@ -6,7 +6,7 @@
 
 from __future__ import print_function
 
-__version__ = "2.13.0"
+__version__ = "2.14.0"
 
 import os
 import sys

src/azure-cli-core/azure/cli/core/_profile.py

@@ -306,7 +306,6 @@ def find_subscriptions_in_vm_with_msi(self, identity_id=None, allow_no_subscript
         # pylint: disable=too-many-statements
 
         import jwt
-        from requests import HTTPError
         from msrestazure.tools import is_valid_resource_id
         from azure.cli.core.adal_authentication import MSIAuthenticationWrapper
         resource = self.cli_ctx.cloud.endpoints.active_directory_resource_id
@@ -317,12 +316,13 @@ def find_subscriptions_in_vm_with_msi(self, identity_id=None, allow_no_subscript
                 identity_type = MsiAccountTypes.user_assigned_resource_id
             else:
                 authenticated = False
+                from azure.cli.core.azclierror import AzureResponseError
                 try:
                     msi_creds = MSIAuthenticationWrapper(resource=resource, client_id=identity_id)
                     identity_type = MsiAccountTypes.user_assigned_client_id
                     authenticated = True
-                except HTTPError as ex:
-                    if ex.response.reason == 'Bad Request' and ex.response.status == 400:
+                except AzureResponseError as ex:
+                    if 'http error: 400, reason: Bad Request' in ex.error_msg:
                         logger.info('Sniff: not an MSI client id')
                     else:
                         raise
@@ -332,8 +332,8 @@ def find_subscriptions_in_vm_with_msi(self, identity_id=None, allow_no_subscript
                         identity_type = MsiAccountTypes.user_assigned_object_id
                         msi_creds = MSIAuthenticationWrapper(resource=resource, object_id=identity_id)
                         authenticated = True
-                    except HTTPError as ex:
-                        if ex.response.reason == 'Bad Request' and ex.response.status == 400:
+                    except AzureResponseError as ex:
+                        if 'http error: 400, reason: Bad Request' in ex.error_msg:
                             logger.info('Sniff: not an MSI object id')
                         else:
                             raise
@@ -1140,7 +1140,7 @@ def __init__(self, password_arg_value, use_cert_sn_issuer=None):
                            'authenticate through a service principal')
         if os.path.isfile(password_arg_value):
             certificate_file = password_arg_value
-            from OpenSSL.crypto import load_certificate, FILETYPE_PEM
+            from OpenSSL.crypto import load_certificate, FILETYPE_PEM, Error
             self.certificate_file = certificate_file
             self.public_certificate = None
             try:
@@ -1154,7 +1154,7 @@ def __init__(self, password_arg_value, use_cert_sn_issuer=None):
                         match = re.search(r'\-+BEGIN CERTIFICATE.+\-+(?P<public>[^-]+)\-+END CERTIFICATE.+\-+',
                                           self.cert_file_string, re.I)
                         self.public_certificate = match.group('public').strip()
-            except UnicodeDecodeError:
+            except (UnicodeDecodeError, Error):
                 raise CLIError('Invalid certificate, please use a valid PEM file.')
         else:
             self.secret = password_arg_value

src/azure-cli-core/azure/cli/core/adal_authentication.py

@@ -93,3 +93,27 @@ class MSIAuthenticationWrapper(MSIAuthentication):
     def get_token(self, *scopes, **kwargs):  # pylint:disable=unused-argument
         self.set_token()
         return AccessToken(self.token['access_token'], int(self.token['expires_on']))
+
+    def set_token(self):
+        import traceback
+        from knack.log import get_logger
+        logger = get_logger(__name__)
+        from azure.cli.core.azclierror import AzureConnectionError, AzureResponseError
+        try:
+            super(MSIAuthenticationWrapper, self).set_token()
+        except requests.exceptions.ConnectionError as err:
+            logger.debug('throw requests.exceptions.ConnectionError when doing MSIAuthentication: \n%s',
+                         traceback.format_exc())
+            raise AzureConnectionError('Failed to connect to MSI. Please make sure MSI is configured correctly '
+                                       'and check the network connection.\nError detail: {}'.format(str(err)))
+        except requests.exceptions.HTTPError as err:
+            logger.debug('throw requests.exceptions.HTTPError when doing MSIAuthentication: \n%s',
+                         traceback.format_exc())
+            raise AzureResponseError('Failed to connect to MSI. Please make sure MSI is configured correctly.\n'
+                                     'Get Token request returned http error: {}, reason: {}'
+                                     .format(err.response.status, err.response.reason))
+        except TimeoutError as err:
+            logger.debug('throw TimeoutError when doing MSIAuthentication: \n%s',
+                         traceback.format_exc())
+            raise AzureConnectionError('MSI endpoint is not responding. Please make sure MSI is configured correctly.\n'
+                                       'Error detail: {}'.format(str(err)))

src/azure-cli-core/azure/cli/core/cloud.py

@@ -163,42 +163,6 @@ def __getattribute__(self, name):
         return val
 
 
-def _get_ossrdbms_resource_id(cloud_name):
-    ossrdbms_mapper = {
-        'AzureCloud': 'https://ossrdbms-aad.database.windows.net',
-        'AzureChinaCloud': 'https://ossrdbms-aad.database.chinacloudapi.cn',
-        'AzureUSGovernment': 'https://ossrdbms-aad.database.usgovcloudapi.net',
-        'AzureGermanCloud': 'https://ossrdbms-aad.database.cloudapi.de'
-    }
-    return ossrdbms_mapper.get(cloud_name, None)
-
-
-def _get_microsoft_graph_resource_id(cloud_name):
-    graph_endpoint_mapper = {
-        'AzureCloud': 'https://graph.microsoft.com/',
-        'AzureChinaCloud': 'https://microsoftgraph.chinacloudapi.cn/',
-        'AzureUSGovernment': 'https://graph.microsoft.us/',
-        'AzureGermanCloud': 'https://graph.microsoft.de/'
-    }
-    return graph_endpoint_mapper.get(cloud_name, None)
-
-
-def _get_storage_sync_endpoint(cloud_name):
-    storage_sync_endpoint_mapper = {
-        'AzureCloud': 'afs.azure.net',
-        'AzureUSGovernment': 'afs.azure.us',
-    }
-    return storage_sync_endpoint_mapper.get(cloud_name, None)
-
-
-def _get_synapse_analytics_endpoint(cloud_name):
-    synapse_analytics_endpoint_mapper = {
-        'AzureCloud': '.dev.azuresynapse.net',
-        'AzureChinaCloud': '.dev.azuresynapse.azure.cn'
-    }
-    return synapse_analytics_endpoint_mapper.get(cloud_name, None)
-
-
 def _get_database_server_endpoint(sql_server_hostname, cloud_name):
     def _concat_db_server_endpoint(db_prefix):
         if cloud_name == 'AzureCloud':
@@ -209,58 +173,18 @@ def _concat_db_server_endpoint(db_prefix):
     return _concat_db_server_endpoint
 
 
-def _get_app_insights_telemetry_channel_resource_id(cloud_name):
-    app_insights_telemetry_channel_resource_id_mapper = {
-        'AzureCloud': 'https://dc.applicationinsights.azure.com/v2/track',
-        'AzureChinaCloud': 'https://dc.applicationinsights.azure.cn/v2/track',
-        'AzureUSGovernment': 'https://dc.applicationinsights.us/v2/track'
-    }
-    return app_insights_telemetry_channel_resource_id_mapper.get(cloud_name, None)
-
-
-def _get_log_analytics_resource_id(cloud_name):
-    log_analytics_resource_id_mapper = {
-        'AzureCloud': 'https://api.loganalytics.io',
-        'AzureChinaCloud': 'https://api.loganalytics.azure.cn',
-        'AzureUSGovernment': 'https://api.loganalytics.us'
-    }
-    return log_analytics_resource_id_mapper.get(cloud_name, None)
-
-
-def _get_app_insights_resource_id(cloud_name):
-    app_insights_resource_id_mapper = {
-        'AzureCloud': 'https://api.applicationinsights.io',
-        'AzureChinaCloud': 'https://api.applicationinsights.azure.cn',
-        'AzureUSGovernment': 'https://api.applicationinsights.us'
-    }
-    return app_insights_resource_id_mapper.get(cloud_name, None)
-
+def _get_endpoint_fallback_value(cloud_name):
+    def _get_cloud_endpoint_fallback_value(endpoint_name):
+        endpoint_mapper = {c.name: c.endpoints.__dict__.get(endpoint_name, None) for c in HARD_CODED_CLOUD_LIST}
+        return endpoint_mapper.get(cloud_name, None)
+    return _get_cloud_endpoint_fallback_value
 
-def _get_synapse_analytics_resource_id(cloud_name):
-    synapse_analytics_resource_id_mapper = {
-        'AzureCloud': 'https://dev.azuresynapse.net',
-        'AzureChinaCloud': 'https://dev.azuresynapse.net'
-    }
-    return synapse_analytics_resource_id_mapper.get(cloud_name, None)
 
-
-def _get_attestation_resource_id(cloud_name):
-    attestation_resource_id_mapper = {
-        'AzureCloud': 'https://attest.azure.net'
-    }
-    return attestation_resource_id_mapper.get(cloud_name, None)
-
-
-def _get_attestation_endpoint(cloud_name):
-    attestation_endpoint_mapper = {
-        'AzureCloud': '.attest.azure.net'
-    }
-    return attestation_endpoint_mapper.get(cloud_name, None)
-
-
-def _get_mhsm_dns_suffix(cloud_name):
-    mhsm_dns_suffix_mapper = {c.name: c.suffixes.mhsm_dns for c in HARD_CODED_CLOUD_LIST}
-    return mhsm_dns_suffix_mapper.get(cloud_name, None)
+def _get_suffix_fallback_value(cloud_name):
+    def _get_cloud_suffix_fallback_value(suffix_name):
+        suffix_mapper = {c.name: c.suffixes.__dict__.get(suffix_name, None) for c in HARD_CODED_CLOUD_LIST}
+        return suffix_mapper.get(cloud_name, None)
+    return _get_cloud_suffix_fallback_value
 
 
 def _convert_arm_to_cli(arm_cloud_metadata_dict):
@@ -291,42 +215,45 @@ def _arm_to_cli_mapper(arm_dict):
     sql_server_hostname = get_suffix('sqlServerHostname', add_dot=True)
     get_db_server_endpoint = _get_database_server_endpoint(sql_server_hostname, arm_dict['name'])
 
+    get_suffix_fallback_value = _get_suffix_fallback_value(arm_dict['name'])
+    get_endpoint_fallback_value = _get_endpoint_fallback_value(arm_dict['name'])
+
     return Cloud(
         arm_dict['name'],
-        endpoints=CloudEndpoints(
+        endpoints=CloudEndpoints(  # please add fallback_value if the endpoint is not added to https://management.azure.com/metadata/endpoints?api-version=2019-05-01 yet
             management=arm_dict['authentication']['audiences'][0],
-            resource_manager=arm_dict['resourceManager'],
-            sql_management=arm_dict['sqlManagement'],
-            batch_resource_id=arm_dict['batch'],
-            gallery=arm_dict['gallery'],
+            resource_manager=get_endpoint('resourceManager'),
+            sql_management=get_endpoint('sqlManagement'),
+            batch_resource_id=get_endpoint('batch'),
+            gallery=get_endpoint('gallery'),
             active_directory=arm_dict['authentication']['loginEndpoint'],
             active_directory_resource_id=arm_dict['authentication']['audiences'][0],
-            active_directory_graph_resource_id=arm_dict['graphAudience'],
-            microsoft_graph_resource_id=_get_microsoft_graph_resource_id(arm_dict['name']),  # change once microsoft_graph_resource_id is fixed in ARM
-            vm_image_alias_doc=arm_dict['vmImageAliasDoc'],
-            media_resource_id=arm_dict['media'],
-            ossrdbms_resource_id=_get_ossrdbms_resource_id(arm_dict['name']),  # change once ossrdbms_resource_id is available via ARM
-            active_directory_data_lake_resource_id=arm_dict['activeDirectoryDataLake'] if 'activeDirectoryDataLake' in arm_dict else None,
-            app_insights_resource_id=get_endpoint('appInsightsResourceId', fallback_value=_get_app_insights_resource_id(arm_dict['name'])),
-            log_analytics_resource_id=get_endpoint('logAnalyticsResourceId', fallback_value=_get_log_analytics_resource_id(arm_dict['name'])),
-            synapse_analytics_resource_id=get_endpoint('synapseAnalyticsResourceId', fallback_value=_get_synapse_analytics_resource_id(arm_dict['name'])),
-            app_insights_telemetry_channel_resource_id=get_endpoint('appInsightsTelemetryChannelResourceId', fallback_value=_get_app_insights_telemetry_channel_resource_id(arm_dict['name'])),
-            attestation_resource_id=get_endpoint('attestationResourceId', fallback_value=_get_attestation_resource_id(arm_dict['name'])),
-            portal=arm_dict['portal'] if 'portal' in arm_dict else None),
+            active_directory_graph_resource_id=get_endpoint('graphAudience'),
+            microsoft_graph_resource_id=get_endpoint('microsoftGraphResourceId', fallback_value=get_endpoint_fallback_value('microsoft_graph_resource_id')),  # change once microsoft_graph_resource_id is fixed in ARM
+            vm_image_alias_doc=get_endpoint('vmImageAliasDoc'),
+            media_resource_id=get_endpoint('media'),
+            ossrdbms_resource_id=get_endpoint('ossrdbmsResourceId', fallback_value=get_endpoint_fallback_value('ossrdbms_resource_id')),  # change once ossrdbms_resource_id is available via ARM
+            active_directory_data_lake_resource_id=get_endpoint('activeDirectoryDataLake'),
+            app_insights_resource_id=get_endpoint('appInsightsResourceId', fallback_value=get_endpoint_fallback_value('app_insights_resource_id')),
+            log_analytics_resource_id=get_endpoint('logAnalyticsResourceId', fallback_value=get_endpoint_fallback_value('log_analytics_resource_id')),
+            synapse_analytics_resource_id=get_endpoint('synapseAnalyticsResourceId', fallback_value=get_endpoint_fallback_value('synapse_analytics_resource_id')),
+            app_insights_telemetry_channel_resource_id=get_endpoint('appInsightsTelemetryChannelResourceId', fallback_value=get_endpoint_fallback_value('app_insights_telemetry_channel_resource_id')),
+            attestation_resource_id=get_endpoint('attestationResourceId', fallback_value=get_endpoint_fallback_value('attestation_resource_id')),
+            portal=get_endpoint('portal')),
         suffixes=CloudSuffixes(
             storage_endpoint=get_suffix('storage'),
-            storage_sync_endpoint=get_suffix('storageSyncEndpointSuffix', fallback_value=_get_storage_sync_endpoint(arm_dict['name'])),
+            storage_sync_endpoint=get_suffix('storageSyncEndpointSuffix', fallback_value=get_suffix_fallback_value('storage_sync_endpoint')),
             keyvault_dns=get_suffix('keyVaultDns', add_dot=True),
-            mhsm_dns=get_suffix('mhsmDns', add_dot=True, fallback_value=_get_mhsm_dns_suffix(arm_dict['name'])),
+            mhsm_dns=get_suffix('mhsmDns', add_dot=True, fallback_value=get_suffix_fallback_value('mhsm_dns')),
             sql_server_hostname=sql_server_hostname,
             mysql_server_endpoint=get_suffix('mysqlServerEndpoint', add_dot=True, fallback_value=get_db_server_endpoint('.mysql')),
             postgresql_server_endpoint=get_suffix('postgresqlServerEndpoint', add_dot=True, fallback_value=get_db_server_endpoint('.postgres')),
             mariadb_server_endpoint=get_suffix('mariadbServerEndpoint', add_dot=True, fallback_value=get_db_server_endpoint('.mariadb')),
             azure_datalake_store_file_system_endpoint=get_suffix('azureDataLakeStoreFileSystem'),
             azure_datalake_analytics_catalog_and_job_endpoint=get_suffix('azureDataLakeAnalyticsCatalogAndJob'),
-            synapse_analytics_endpoint=get_suffix('synapseAnalytics', add_dot=True, fallback_value=_get_synapse_analytics_endpoint(arm_dict['name'])),
+            synapse_analytics_endpoint=get_suffix('synapseAnalytics', add_dot=True, fallback_value=get_suffix_fallback_value('synapse_analytics_endpoint')),
             acr_login_server_endpoint=get_suffix('acrLoginServer', add_dot=True),
-            attestation_endpoint=get_suffix('attestationEndpoint', add_dot=True, fallback_value=_get_attestation_endpoint(arm_dict['name']))))
+            attestation_endpoint=get_suffix('attestationEndpoint', add_dot=True, fallback_value=get_suffix_fallback_value('attestation_endpoint'))))
 
 
 class Cloud:  # pylint: disable=too-few-public-methods

src/azure-cli-core/azure/cli/core/parser.py

@@ -223,7 +223,7 @@ def get_examples(self, command):
     def enable_autocomplete(self):
         argcomplete.autocomplete = AzCompletionFinder()
         argcomplete.autocomplete(self, validator=lambda c, p: c.lower().startswith(p.lower()),
-                                 default_completer=lambda _: ())
+                                 default_completer=lambda *args, **kwargs: ())
 
     def _get_failure_recovery_arguments(self, action=None):
         # Strip the leading "az " and any extraneous whitespace.

src/azure-cli-core/azure/cli/core/telemetry.py

@@ -22,6 +22,10 @@
 AZURE_CLI_PREFIX = 'Context.Default.AzureCLI.'
 DEFAULT_INSTRUMENTATION_KEY = 'c4395b75-49cc-422c-bc95-c7d51aef5d46'
 CORRELATION_ID_PROP_NAME = 'Reserved.DataModel.CorrelationId'
+# Put a config section or key (section.name) in the allowed set to allow recording the config
+# values in the section or for the key with 'az config set'
+ALLOWED_CONFIG_SECTIONS_OR_KEYS = {'auto-upgrade', 'extension', 'core', 'logging.enable_log_file',
+                                   'output.show_survey_link'}
 
 
 class TelemetrySession:  # pylint: disable=too-many-instance-attributes
@@ -136,6 +140,7 @@ def _get_base_properties(self):
             'Context.Default.VS.Core.Machine.Id': _get_hash_machine_id(),
             'Context.Default.VS.Core.OS.Type': platform.system().lower(),  # eg. darwin, windows
             'Context.Default.VS.Core.OS.Version': platform.version().lower(),  # eg. 10.0.14942
+            'Context.Default.VS.Core.OS.Platform': platform.platform().lower(),  # eg. windows-10-10.0.19041-sp0
             'Context.Default.VS.Core.User.Id': _get_installation_id(),
             'Context.Default.VS.Core.User.IsMicrosoftInternal': 'False',
             'Context.Default.VS.Core.User.IsOptedIn': 'True',
@@ -343,10 +348,25 @@ def set_user_fault(summary=None):
 
 @decorators.suppress_all_exceptions()
 def set_debug_info(key, info):
+    if key == 'ConfigSet':
+        info = _process_config_set_debug_info(info)
+
     debug_info = '{}: {}'.format(key, info)
     _session.debug_info.append(debug_info)
 
 
+@decorators.suppress_all_exceptions()
+def _process_config_set_debug_info(info):
+    processed_info = []
+    # info is a list of tuples
+    for key, section, value in info:
+        if section in ALLOWED_CONFIG_SECTIONS_OR_KEYS or key in ALLOWED_CONFIG_SECTIONS_OR_KEYS:
+            processed_info.append('{}={}'.format(key, value))
+        else:
+            processed_info.append('{}={}'.format(key, '***' if value else value))
+    return ' '.join(processed_info)
+
+
 @decorators.suppress_all_exceptions()
 def set_application(application, arg_complete_env_name):
     _session.application, _session.arg_complete_env_name = application, arg_complete_env_name