Making regression3 pass

gcloud/connection.py

@@ -16,6 +16,7 @@
 
 import json
 from pkg_resources import get_distribution
+import six
 from six.moves.urllib.parse import urlencode # pylint: disable=F0401
 
 import httplib2
@@ -295,6 +296,8 @@ def api_request(self, method, path, query_params=None,
  content_type = response.get('content-type', '')
  if not content_type.startswith('application/json'):
  raise TypeError('Expected JSON, got %s' % content_type)
+ if isinstance(content, six.binary_type):
+ content = content.decode('utf-8')
  return json.loads(content)
 
  return content

gcloud/credentials.py

@@ -181,6 +181,8 @@ def _get_signed_query_params(credentials, expiration, signature_string):
  pem_key = _get_pem_key(credentials)
  # Sign the string with the RSA key.
  signer = PKCS1_v1_5.new(pem_key)
+ if not isinstance(signature_string, six.binary_type):
+ signature_string = signature_string.encode('utf-8')
  signature_hash = SHA256.new(signature_string)
  signature_bytes = signer.sign(signature_hash)
  signature = base64.b64encode(signature_bytes)

gcloud/datastore/test_connection.py

@@ -152,7 +152,7 @@ def test__request_not_200(self):
  METHOD = 'METHOD'
  DATA = 'DATA'
  conn = self._makeOne()
- conn._http = Http({'status': '400'}, 'Entity value is indexed.')
+ conn._http = Http({'status': '400'}, b'Entity value is indexed.')
  with self.assertRaises(BadRequest) as e:
  conn._request(DATASET_ID, METHOD, DATA)
  expected_message = '400 Entity value is indexed.'

gcloud/exceptions.py

@@ -18,6 +18,7 @@
 """
 
 import json
+import six
 
 _HTTP_CODE_TO_EXCEPTION = {} # populated at end of module
 
@@ -171,18 +172,18 @@ def make_exception(response, content, use_json=True):
  :rtype: instance of :class:`GCloudError`, or a concrete subclass.
  :returns: Exception specific to the error response.
  """
-message = content
-errors = ()
+if isinstance(content, six.binary_type):
+ content = content.decode('utf-8')
 
- if isinstance(content, str):
+ if isinstance(content, six.string_types):
  if use_json:
  payload = json.loads(content)
  else:
- payload = {}
+ payload = {'message': content}
  else:
  payload = content
 
- message = payload.get('message', message)
+ message = payload.get('message', '')
  errors = payload.get('error', {}).get('errors', ())
 
  try:

gcloud/storage/batch.py

@@ -170,10 +170,25 @@ def __exit__(self, exc_type, exc_val, exc_tb):
 def _unpack_batch_response(response, content):
  """Convert response, content -> [(status, reason, payload)]."""
  parser = Parser()
- faux_message = ('Content-Type: %s\nMIME-Version: 1.0\n\n%s' %
- (response['content-type'], content))
-
- message = parser.parsestr(faux_message)
+ # We coerce to bytes to get consitent concat across
+ # Py2 and Py3. Percent formatting is insufficient since
+ # it includes the b in Py3.
+ if not isinstance(content, six.binary_type):
+ content = content.encode('utf-8')
+ content_type = response['content-type']
+ if not isinstance(content_type, six.binary_type):
+ content_type = content_type.encode('utf-8')
+ faux_message = b''.join([
+ b'Content-Type: ',
+ content_type,
+ b'\nMIME-Version: 1.0\n\n',
+ content,
+ ])
+
+ if six.PY2:
+ message = parser.parsestr(faux_message)
+ else: # pragma: NO COVER Python3
+ message = parser.parsestr(faux_message.decode('utf-8'))
 
  if not isinstance(message._payload, list):
  raise ValueError('Bad response: not multi-part')

gcloud/storage/test_api.py

@@ -34,7 +34,7 @@ def test_miss(self):
  ])
  http = conn._http = Http(
  {'status': '404', 'content-type': 'application/json'},
- '{}',
+b'{}',
  )
  bucket = self._callFUT(NONESUCH, connection=conn)
  self.assertEqual(bucket, None)
@@ -56,7 +56,7 @@ def _lookup_bucket_hit_helper(self, use_default=False):
  ])
  http = conn._http = Http(
  {'status': '200', 'content-type': 'application/json'},
- '{"name": "%s"}' % BLOB_NAME,
+ '{{"name": "{0}"}}'.format(BLOB_NAME).encode('utf-8'),
  )
 
  if use_default:
@@ -96,7 +96,7 @@ def test_empty(self):
  ])
  http = conn._http = Http(
  {'status': '200', 'content-type': 'application/json'},
- '{}',
+b'{}',
  )
  buckets = list(self._callFUT(PROJECT, conn))
  self.assertEqual(len(buckets), 0)
@@ -117,7 +117,8 @@ def _get_all_buckets_non_empty_helper(self, project, use_default=False):
  ])
  http = conn._http = Http(
  {'status': '200', 'content-type': 'application/json'},
- '{"items": [{"name": "%s"}]}' % BUCKET_NAME,
+ '{{"items": [{{"name": "{0}"}}]}}'.format(BUCKET_NAME)
+ .encode('utf-8'),
  )
 
  if use_default:
@@ -159,7 +160,7 @@ def test_miss(self):
  ])
  http = conn._http = Http(
  {'status': '404', 'content-type': 'application/json'},
- '{}',
+b'{}',
  )
  self.assertRaises(NotFound, self._callFUT, NONESUCH, connection=conn)
  self.assertEqual(http._called_with['method'], 'GET')
@@ -180,7 +181,7 @@ def _get_bucket_hit_helper(self, use_default=False):
  ])
  http = conn._http = Http(
  {'status': '200', 'content-type': 'application/json'},
- '{"name": "%s"}' % BLOB_NAME,
+ '{{"name": "{0}"}}'.format(BLOB_NAME).encode('utf-8'),
  )
 
  if use_default:
@@ -224,7 +225,7 @@ def _create_bucket_success_helper(self, project, use_default=False):
  ])
  http = conn._http = Http(
  {'status': '200', 'content-type': 'application/json'},
- '{"name": "%s"}' % BLOB_NAME,
+ '{{"name": "{0}"}}'.format(BLOB_NAME).encode('utf-8'),
  )
 
  if use_default:

gcloud/storage/test_batch.py

@@ -347,7 +347,32 @@ def test_as_context_mgr_w_error(self):
  self.assertEqual(len(batch._responses), 0)
 
 
-_THREE_PART_MIME_RESPONSE = """\
+class Test__unpack_batch_response(unittest2.TestCase):
+
+ def _callFUT(self, response, content):
+ from gcloud.storage.batch import _unpack_batch_response
+ return _unpack_batch_response(response, content)
+
+ def test_bytes(self):
+ RESPONSE = {'content-type': b'multipart/mixed; boundary="DEADBEEF="'}
+ CONTENT = _THREE_PART_MIME_RESPONSE
+ result = list(self._callFUT(RESPONSE, CONTENT))
+ self.assertEqual(len(result), 3)
+ self.assertEqual(result[0], ('200', 'OK', {u'bar': 2, u'foo': 1}))
+ self.assertEqual(result[1], ('200', 'OK', {u'foo': 1, u'bar': 3}))
+ self.assertEqual(result[2], ('204', 'No Content', ''))
+
+ def test_unicode(self):
+ RESPONSE = {'content-type': u'multipart/mixed; boundary="DEADBEEF="'}
+ CONTENT = _THREE_PART_MIME_RESPONSE.decode('utf-8')
+ result = list(self._callFUT(RESPONSE, CONTENT))
+ self.assertEqual(len(result), 3)
+ self.assertEqual(result[0], ('200', 'OK', {u'bar': 2, u'foo': 1}))
+ self.assertEqual(result[1], ('200', 'OK', {u'foo': 1, u'bar': 3}))
+ self.assertEqual(result[2], ('204', 'No Content', ''))
+
+
+_THREE_PART_MIME_RESPONSE = b"""\
 --DEADBEEF=
 Content-Type: application/http
 Content-ID: <response-8a09ca85-8d1d-4f45-9eb0-da8e8b07ec83+1>

gcloud/test_connection.py

@@ -161,12 +161,12 @@ def test__make_request_no_data_no_content_type_no_headers(self):
  URI = 'http://example.com/test'
  http = conn._http = _Http(
  {'status': '200', 'content-type': 'text/plain'},
- '',
+b'',
  )
  headers, content = conn._make_request('GET', URI)
  self.assertEqual(headers['status'], '200')
  self.assertEqual(headers['content-type'], 'text/plain')
- self.assertEqual(content, '')
+ self.assertEqual(content, b'')
  self.assertEqual(http._called_with['method'], 'GET')
  self.assertEqual(http._called_with['uri'], URI)
  self.assertEqual(http._called_with['body'], None)
@@ -182,7 +182,7 @@ def test__make_request_w_data_no_extra_headers(self):
  URI = 'http://example.com/test'
  http = conn._http = _Http(
  {'status': '200', 'content-type': 'text/plain'},
- '',
+b'',
  )
  conn._make_request('GET', URI, {}, 'application/json')
  self.assertEqual(http._called_with['method'], 'GET')
@@ -201,7 +201,7 @@ def test__make_request_w_extra_headers(self):
  URI = 'http://example.com/test'
  http = conn._http = _Http(
  {'status': '200', 'content-type': 'text/plain'},
- '',
+b'',
  )
  conn._make_request('GET', URI, headers={'X-Foo': 'foo'})
  self.assertEqual(http._called_with['method'], 'GET')
@@ -226,7 +226,7 @@ def test_api_request_defaults(self):
  ])
  http = conn._http = _Http(
  {'status': '200', 'content-type': 'application/json'},
- '{}',
+b'{}',
  )
  self.assertEqual(conn.api_request('GET', PATH), {})
  self.assertEqual(http._called_with['method'], 'GET')
@@ -243,7 +243,7 @@ def test_api_request_w_non_json_response(self):
  conn = self._makeMockOne()
  conn._http = _Http(
  {'status': '200', 'content-type': 'text/plain'},
- 'CONTENT',
+b'CONTENT',
  )
 
  self.assertRaises(TypeError, conn.api_request, 'GET', '/')
@@ -252,18 +252,18 @@ def test_api_request_wo_json_expected(self):
  conn = self._makeMockOne()
  conn._http = _Http(
  {'status': '200', 'content-type': 'text/plain'},
- 'CONTENT',
+b'CONTENT',
  )
  self.assertEqual(conn.api_request('GET', '/', expect_json=False),
- 'CONTENT')
+b'CONTENT')
 
  def test_api_request_w_query_params(self):
  from six.moves.urllib.parse import parse_qsl
  from six.moves.urllib.parse import urlsplit
  conn = self._makeMockOne()
  http = conn._http = _Http(
  {'status': '200', 'content-type': 'application/json'},
- '{}',
+b'{}',
  )
  self.assertEqual(conn.api_request('GET', '/', {'foo': 'bar'}), {})
  self.assertEqual(http._called_with['method'], 'GET')
@@ -302,7 +302,7 @@ def test_api_request_w_data(self):
  ])
  http = conn._http = _Http(
  {'status': '200', 'content-type': 'application/json'},
- '{}',
+b'{}',
  )
  self.assertEqual(conn.api_request('POST', '/', data=DATA), {})
  self.assertEqual(http._called_with['method'], 'POST')
@@ -321,7 +321,7 @@ def test_api_request_w_404(self):
  conn = self._makeMockOne()
  conn._http = _Http(
  {'status': '404', 'content-type': 'text/plain'},
- '{}'
+b'{}'
  )
  self.assertRaises(NotFound, conn.api_request, 'GET', '/')
 
@@ -330,10 +330,35 @@ def test_api_request_w_500(self):
  conn = self._makeMockOne()
  conn._http = _Http(
  {'status': '500', 'content-type': 'text/plain'},
- '{}',
+b'{}',
  )
  self.assertRaises(InternalServerError, conn.api_request, 'GET', '/')
 
+ def test_api_request_non_binary_response(self):
+ conn = self._makeMockOne()
+ http = conn._http = _Http(
+ {'status': '200', 'content-type': 'application/json'},
+ u'{}',
+ )
+ result = conn.api_request('GET', '/')
+ # Intended to emulate self.mock_template
+ URI = '/'.join([
+ conn.API_BASE_URL,
+ 'mock',
+ conn.API_VERSION,
+ '',
+ ])
+ self.assertEqual(result, {})
+ self.assertEqual(http._called_with['method'], 'GET')
+ self.assertEqual(http._called_with['uri'], URI)
+ self.assertEqual(http._called_with['body'], None)
+ expected_headers = {
+ 'Accept-Encoding': 'gzip',
+ 'Content-Length': 0,
+ 'User-Agent': conn.USER_AGENT,
+ }
+ self.assertEqual(http._called_with['headers'], expected_headers)
+
 
 class _Http(object):
 

gcloud/test_credentials.py

@@ -176,11 +176,13 @@ def _get_pem_key(credentials):
  SIGNATURE_STRING = 'dummy_signature'
  with _Monkey(MUT, RSA=rsa, PKCS1_v1_5=pkcs_v1_5,
  SHA256=sha256, _get_pem_key=_get_pem_key):
- self.assertRaises(NameError, self._callFUT,
+ self.assertRaises(UnboundLocalError, self._callFUT,
  BAD_CREDENTIALS, EXPIRATION, SIGNATURE_STRING)
 
- def _run_test_with_credentials(self, credentials, account_name):
+ def _run_test_with_credentials(self, credentials, account_name,
+ signature_string=None):
  import base64
+ import six
  from gcloud._testing import _Monkey
  from gcloud import credentials as MUT
 
@@ -190,7 +192,7 @@ def _run_test_with_credentials(self, credentials, account_name):
  sha256 = _SHA256()
 
  EXPIRATION = '100'
- SIGNATURE_STRING = b'dummy_signature'
+ SIGNATURE_STRING = signature_string or b'dummy_signature'
  with _Monkey(MUT, crypt=crypt, RSA=rsa, PKCS1_v1_5=pkcs_v1_5,
  SHA256=sha256):
  result = self._callFUT(credentials, EXPIRATION, SIGNATURE_STRING)
@@ -199,7 +201,12 @@ def _run_test_with_credentials(self, credentials, account_name):
  self.assertEqual(crypt._private_key_text,
  base64.b64encode(b'dummy_private_key_text'))
  self.assertEqual(crypt._private_key_password, 'notasecret')
- self.assertEqual(sha256._signature_string, SIGNATURE_STRING)
+ # sha256._signature_string is always bytes.
+ if isinstance(SIGNATURE_STRING, six.binary_type):
+ self.assertEqual(sha256._signature_string, SIGNATURE_STRING)
+ else:
+ self.assertEqual(sha256._signature_string,
+ SIGNATURE_STRING.encode('utf-8'))
  SIGNED = base64.b64encode(b'DEADBEEF')
  expected_query = {
  'Expires': EXPIRATION,
@@ -217,6 +224,17 @@ def test_signed_jwt_for_p12(self):
  ACCOUNT_NAME, b'dummy_private_key_text', scopes)
  self._run_test_with_credentials(credentials, ACCOUNT_NAME)
 
+ def test_signature_non_bytes(self):
+ from oauth2client import client
+
+ scopes = []
+ ACCOUNT_NAME = 'dummy_service_account_name'
+ SIGNATURE_STRING = u'dummy_signature'
+ credentials = client.SignedJwtAssertionCredentials(
+ ACCOUNT_NAME, b'dummy_private_key_text', scopes)
+ self._run_test_with_credentials(credentials, ACCOUNT_NAME,
+ signature_string=SIGNATURE_STRING)
+
  def test_service_account_via_json_key(self):
  from oauth2client import service_account
  from gcloud._testing import _Monkey

gcloud/test_exceptions.py

@@ -55,7 +55,7 @@ def _callFUT(self, response, content):
  def test_hit_w_content_as_str(self):
  from gcloud.exceptions import NotFound
  response = _Response(404)
- content = '{"message": "Not Found"}'
+ content = b'{"message": "Not Found"}'
  exception = self._callFUT(response, content)
  self.assertTrue(isinstance(exception, NotFound))
  self.assertEqual(exception.message, 'Not Found')