Merge branch 'release-0.0.8'

danielgtaylor · danielgtaylor · commit e8e0fcf2896a · 2015-02-10T16:49:09.000-08:00
diff --git a/CHANGELOG.rst b/CHANGELOG.rst
@@ -1,14 +1,34 @@
 Changelog
 =========
 
+0.0.8 - 2015-02-10
+------------------
+
+* bugfix:Resources: Fix Amazon S3 resource identifier order.
+  (`issue 62 <https://github.com/boto/boto3/pull/62>`__)
+* bugfix:Resources: Fix collection resource hydration path.
+  (`issue 61 <https://github.com/boto/boto3/pull/61>`__)
+* bugfix:Resources: Re-enable service-level access to all resources,
+  allowing e.g. ``obj = s3.Object('bucket', 'key')``.
+  (`issue 60 <https://github.com/boto/boto3/pull/60>`__)
+* feature:Botocore: Update to Botocore 0.87.0
+
+  * Add support for Amazon DynamoDB secondary index scanning.
+  * Upgrade to ``requests`` 2.5.1.
+  * Add support for anonymous (unsigned) clients.
+    (`botocore issue 448 <https://github.com/boto/botocore/pull/448>`__)
+
 0.0.7 - 2015-02-05
 ------------------
 
 * feature:Resources: Enable support for Amazon Glacier.
 * feature:Resources: Support plural references and nested JMESPath
   queries for data members when building parameters and identifiers.
   (`issue 52 <https://github.com/boto/boto3/pull/52>`__)
-* feature:Resources: Update to the latest resource JSON format.
+* feature:Resources: Update to the latest resource JSON format. This is
+  a **backward-incompatible** change as not all resources are exposed
+  at the service level anymore. For example, ``s3.Object('bucket', 'key')``
+  is now ``s3.Bucket('bucket').Object('key')``.
   (`issue 51 <https://github.com/boto/boto3/pull/51>`__)
 * feature:Resources: Make ``resource.meta`` a proper object. This allows
   you to do things like ``resource.meta.client``. This is a **backward-
diff --git a/LICENSE b/LICENSE
@@ -1,4 +1,4 @@
-Copyright 2013-2014 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+Copyright 2013-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved.
 
 Licensed under the Apache License, Version 2.0 (the "License"). You
 may not use this file except in compliance with the License. A copy of
diff --git a/README.rst b/README.rst
@@ -40,22 +40,30 @@ pull requests on this repository. Thanks!
 
 Quick Start
 -----------
-First, install the library and set a default region::
+First, install the library and set a default region:
+
+.. code-block:: sh
 
     $ pip install boto3
 
-Next, set up credentials (in e.g. ``~/.aws/credentials``)::
+Next, set up credentials (in e.g. ``~/.aws/credentials``):
+
+.. code-block:: ini
 
     [default]
     aws_access_key_id = YOUR_KEY
     aws_secret_access_key = YOUR_SECRET
 
-Then, set up a default region (in e.g. ``~/.aws/config``)::
+Then, set up a default region (in e.g. ``~/.aws/config``):
+
+.. code-block:: ini
 
     [default]
     region=us-east-1
 
-Then, from a Python interpreter::
+Then, from a Python interpreter:
+
+.. code-block:: python
 
     >>> import boto3
     >>> s3 = boto3.resource('s3')
@@ -69,7 +77,9 @@ Getting Started
 ~~~~~~~~~~~~~~~
 Assuming that you have Python and ``virtualenv`` installed, set up your
 environment and install the required dependencies like this instead of
-the ``pip install boto3`` defined above::
+the ``pip install boto3`` defined above:
+
+.. code-block:: sh
 
     $ git clone https://github.com/boto/boto3.git
     $ cd boto3
@@ -85,20 +95,26 @@ You can run tests in all supported Python versions using ``tox``. Be default,
 it will run all of the unit tests, but you can also specify your own
 ``nosetests`` options. Note that this requires that you have all supported
 versions of Python installed, otherwise you must pass ``-e`` or run the
-``nosetests`` command directly::
+``nosetests`` command directly:
+
+.. code-block:: sh
 
     $ tox
     $ tox tests/unit/test_session.py
     $ tox -e py26,py33 tests/integration
 
-You can also run individual tests with your default Python version::
+You can also run individual tests with your default Python version:
+
+.. code-block:: sh
 
     $ nosetests tests/unit
 
 Generating Documentation
 ~~~~~~~~~~~~~~~~~~~~~~~~
 Sphinx is used for documentation. You can generate HTML locally with the
-following::
+following:
+
+.. code-block:: sh
 
     $ pip install sphinx sphinx_rtd_theme
     $ cd docs
diff --git a/boto3/__init__.py b/boto3/__init__.py
@@ -17,7 +17,7 @@
 
 
 __author__ = 'Amazon Web Services'
-__version__ = '0.0.7'
+__version__ = '0.0.8'
 
 
 # The default Boto3 session; autoloaded when needed.
diff --git a/boto3/data/resources/cloudformation-2010-05-15.resources.json b/boto3/data/resources/cloudformation-2010-05-15.resources.json
@@ -94,7 +94,7 @@
             "type": "StackResource",
             "identifiers": [
               { "target": "StackName", "source": "identifier", "name": "Name" },
-              { "target": "StackName", "source": "input" }
+              { "target": "LogicalId", "source": "input" }
             ]
           }
         }
@@ -135,7 +135,7 @@
     },
     "StackResource": {
       "identifiers": [
-        { "name": "LogicalId" },
+        { "name": "LogicalId", "memberName": "LogicalResourceId" },
         { "name": "StackName" }
       ],
       "shape": "StackResourceDetail",
@@ -162,7 +162,7 @@
     },
     "StackResourceSummary": {
       "identifiers": [
-        { "name": "LogicalId" },
+        { "name": "LogicalId", "memberName": "LogicalResourceId" },
         { "name": "StackName" }
       ],
       "shape": "StackResourceSummary",
diff --git a/boto3/data/resources/glacier-2012-06-01.resources.json b/boto3/data/resources/glacier-2012-06-01.resources.json
@@ -110,11 +110,11 @@
             ]
           }
         },
-        "InitiateArchiveRetreival": {
+        "InitiateArchiveRetrieval": {
           "request": {
             "operation": "InitiateJob",
             "params": [
-              { "target": "vaultName", "source": "identifier", "name": "Name" },
+              { "target": "vaultName", "source": "identifier", "name": "VaultName" },
               { "target": "accountId", "source": "identifier", "name": "AccountId" },
               { "target": "jobParameters.Type", "source": "string", "value": "archive-retrieval" },
               { "target": "jobParameters.ArchiveId", "source": "identifier", "name": "Id" }
@@ -125,7 +125,7 @@
             "identifiers": [
               { "target": "Id", "source": "response", "path": "jobId" },
               { "target": "AccountId", "source": "identifier", "name": "AccountId" },
-              { "target": "VaultName", "source": "identifier", "name": "Name" }
+              { "target": "VaultName", "source": "identifier", "name": "VaultName" }
             ]
           }
         }
diff --git a/boto3/data/resources/iam-2010-05-08.resources.json b/boto3/data/resources/iam-2010-05-08.resources.json
@@ -376,7 +376,7 @@
           "request": {
             "operation": "UpdateAssumeRolePolicy",
             "params": [
-              { "target": "RoleName", "source": "identifier", "name": "Name" }
+              { "target": "RoleName", "source": "identifier", "name": "RoleName" }
             ]
           }
         }
diff --git a/boto3/data/resources/s3-2006-03-01.resources.json b/boto3/data/resources/s3-2006-03-01.resources.json
@@ -698,8 +698,8 @@
             "type": "MultipartUploadPart",
             "identifiers": [
               { "target": "BucketName", "source": "identifier", "name": "BucketName" },
-              { "target": "MultipartUploadId", "source": "identifier", "name": "Id" },
               { "target": "ObjectKey", "source": "identifier", "name": "ObjectKey" },
+              { "target": "MultipartUploadId", "source": "identifier", "name": "Id" },
               { "target": "PartNumber", "source": "input" }
             ]
           }
@@ -770,8 +770,8 @@
             "type": "MultipartUpload",
             "identifiers": [
               { "target": "BucketName", "source": "identifier", "name": "BucketName" },
-              { "target": "Id", "source": "identifier", "name": "MultipartUploadId" },
-              { "target": "ObjectKey", "source": "identifier", "name": "ObjectKey" }
+              { "target": "ObjectKey", "source": "identifier", "name": "ObjectKey" },
+              { "target": "Id", "source": "identifier", "name": "MultipartUploadId" }
             ]
           }
         }
diff --git a/boto3/data/resources/sns-2010-03-31.resources.json b/boto3/data/resources/sns-2010-03-31.resources.json
@@ -110,7 +110,6 @@
           "resource": {
             "type": "PlatformEndpoint",
             "identifiers": [
-              { "target": "PlatformApplicationArn", "source": "identifier", "name": "Arn" },
               { "target": "Arn", "source": "response", "path": "EndpointArn" }
             ]
           }
diff --git a/boto3/resources/collection.py b/boto3/resources/collection.py
@@ -312,7 +312,7 @@ def __init__(self, model, parent, factory, resource_defs,
         operation_name = self._model.request.operation
         self._parent = parent
 
-        search_path = model.path
+        search_path = model.resource.path
         self._handler = ResourceHandler(search_path, factory, resource_defs,
             service_model, model.resource, operation_name)
 
diff --git a/boto3/resources/factory.py b/boto3/resources/factory.py
@@ -188,7 +188,7 @@ def _load_has_relations(self, attrs, service_name, resource_name,
             # This is a sub-resource class you can create
             # by passing in an identifier, e.g. s3.Bucket(name).
             name = subresource.resource.type
-            attrs[name] = self._create_class_partial(
+            attrs[subresource.name] = self._create_class_partial(
                 name, subresource, service_name, resource_name, model,
                 resource_defs, service_model)
 
diff --git a/boto3/resources/model.py b/boto3/resources/model.py
@@ -312,6 +312,69 @@ def batch_actions(self):
 
         return actions
 
+    def _get_has_definition(self):
+        """
+        Get a ``has`` relationship definition from a model, where the
+        service resource model is treated special in that it contains
+        a relationship to every resource defined for the service. This
+        allows things like ``s3.Object('bucket-name', 'key')`` to
+        work even though the JSON doesn't define it explicitly.
+
+        :rtype: dict
+        :return: Mapping of names to subresource and reference
+                 definitions.
+        """
+        if self.name not in self._resource_defs:
+            # This is the service resource, so let us expose all of
+            # the defined resources as subresources.
+            definition = {}
+
+            for name, resource_def in self._resource_defs.items():
+                # It's possible for the service to have renamed a
+                # resource or to have defined multiple names that
+                # point to the same resource type, so we need to
+                # take that into account.
+                found = False
+                has_items = self._definition.get('has', {}).items()
+                for has_name, has_def in has_items:
+                    if has_def.get('resource', {}).get('type') == name:
+                        definition[has_name] = has_def
+                        found = True
+
+                if not found:
+                    # Create a relationship definition and attach it
+                    # to the model, such that all identifiers must be
+                    # supplied by the user. It will look something like:
+                    #
+                    # {
+                    #   'resource': {
+                    #     'type': 'ResourceName',
+                    #     'identifiers': [
+                    #       {'target': 'Name1', 'source': 'input'},
+                    #       {'target': 'Name2', 'source': 'input'},
+                    #       ...
+                    #     ]
+                    #   }
+                    # }
+                    #
+                    fake_has = {
+                        'resource': {
+                            'type': name,
+                            'identifiers': []
+                        }
+                    }
+
+                    for identifier in resource_def.get('identifiers', []):
+                        fake_has['resource']['identifiers'].append({
+                            'target': identifier['name'], 'source': 'input'
+                        })
+
+                    definition[name] = fake_has
+        else:
+            definition = self._definition.get('has', {})
+
+        return definition
+
     def _get_related_resources(self, subresources):
         """
         Get a list of sub-resources or references.
@@ -323,7 +386,7 @@ def _get_related_resources(self, subresources):
         """
         resources = []
 
-        for name, definition in self._definition.get('has', {}).items():
+        for name, definition in self._get_has_definition().items():
             action = Action(name, definition, self._resource_defs)
 
             data_required = False
diff --git a/setup.py b/setup.py
@@ -26,7 +26,7 @@ def get_version():
 ]
 
 requires = [
-    'botocore==0.86.0',
+    'botocore==0.87.0',
     'bcdoc==0.12.2',
     'jmespath==0.6.1',
 ]
diff --git a/tests/integration/test_s3.py b/tests/integration/test_s3.py
@@ -57,6 +57,9 @@ def test_s3(self):
         # Lazy-loaded attribute
         self.assertEqual(12, obj.content_length)
 
+        # Load a similar attribute from the collection response
+        self.assertEqual(12, list(bucket.objects.all())[0].size)
+
         # Perform a resource action with a low-level response
         self.assertEqual(b'hello, world',
                          obj.get()['Body'].read())
@@ -82,4 +85,39 @@ def test_s3_resource_waiter(self):
         obj.wait_until_exists()
 
         # List objects and make sure ours is present
-        self.assertIn('test.txt', [o.key for o in bucket.objects.all()])        
+        self.assertIn('test.txt', [o.key for o in bucket.objects.all()])
+
+    def test_can_create_object_directly(self):
+        obj = self.s3.Object(self.bucket_name, 'test.txt')
+
+        self.assertEqual(obj.bucket_name, self.bucket_name)
+        self.assertEqual(obj.key, 'test.txt')
+
+    def test_s3_multipart(self):
+        # Create the bucket
+        bucket = self.create_bucket_resource(self.bucket_name)
+        bucket.wait_until_exists()
+
+        # Create the multipart upload
+        mpu = bucket.Object('mp-test.txt').initiate_multipart_upload()
+        self.addCleanup(mpu.abort)
+
+        # Create and upload a part
+        part = mpu.Part(1)
+        response = part.upload(Body='hello, world!')
+
+        # Complete the upload, which requires info on all of the parts
+        part_info = {
+            'Parts': [
+                {
+                    'PartNumber': 1,
+                    'ETag': response['ETag']
+                }
+            ]
+        }
+
+        mpu.complete(MultipartUpload=part_info)
+        self.addCleanup(bucket.Object('mp-test.txt').delete)
+
+        contents = bucket.Object('mp-test.txt').get()['Body'].read()
+        self.assertEqual(contents, b'hello, world!')
diff --git a/tests/unit/resources/test_factory.py b/tests/unit/resources/test_factory.py
diff --git a/tests/unit/resources/test_model.py b/tests/unit/resources/test_model.py

Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,4 @@`
`1`		`-Copyright 2013-2014 Amazon.com, Inc. or its affiliates. All Rights Reserved.`
	`1`	`+Copyright 2013-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved.`
`2`	`2`
`3`	`3`	`Licensed under the Apache License, Version 2.0 (the "License"). You`
`4`	`4`	`may not use this file except in compliance with the License. A copy of`
Original file line number	Diff line number	Diff line change
`@@ -110,11 +110,11 @@`
`110`	`110`	`]`
`111`	`111`	`}`
`112`	`112`	`},`
`113`		`- "InitiateArchiveRetreival": {`
	`113`	`+ "InitiateArchiveRetrieval": {`
`114`	`114`	`"request": {`
`115`	`115`	`"operation": "InitiateJob",`
`116`	`116`	`"params": [`
`117`		`- { "target": "vaultName", "source": "identifier", "name": "Name" },`
	`117`	`+ { "target": "vaultName", "source": "identifier", "name": "VaultName" },`
`118`	`118`	`{ "target": "accountId", "source": "identifier", "name": "AccountId" },`
`119`	`119`	`{ "target": "jobParameters.Type", "source": "string", "value": "archive-retrieval" },`
`120`	`120`	`{ "target": "jobParameters.ArchiveId", "source": "identifier", "name": "Id" }`
`@@ -125,7 +125,7 @@`
`125`	`125`	`"identifiers": [`
`126`	`126`	`{ "target": "Id", "source": "response", "path": "jobId" },`
`127`	`127`	`{ "target": "AccountId", "source": "identifier", "name": "AccountId" },`
`128`		`- { "target": "VaultName", "source": "identifier", "name": "Name" }`
	`128`	`+ { "target": "VaultName", "source": "identifier", "name": "VaultName" }`
`129`	`129`	`]`
`130`	`130`	`}`
`131`	`131`	`}`
Original file line number	Diff line number	Diff line change
`@@ -376,7 +376,7 @@`
`376`	`376`	`"request": {`
`377`	`377`	`"operation": "UpdateAssumeRolePolicy",`
`378`	`378`	`"params": [`
`379`		`- { "target": "RoleName", "source": "identifier", "name": "Name" }`
	`379`	`+ { "target": "RoleName", "source": "identifier", "name": "RoleName" }`
`380`	`380`	`]`
`381`	`381`	`}`
`382`	`382`	`}`
Original file line number	Diff line number	Diff line change
`@@ -698,8 +698,8 @@`
`698`	`698`	`"type": "MultipartUploadPart",`
`699`	`699`	`"identifiers": [`
`700`	`700`	`{ "target": "BucketName", "source": "identifier", "name": "BucketName" },`
`701`		`- { "target": "MultipartUploadId", "source": "identifier", "name": "Id" },`
`702`	`701`	`{ "target": "ObjectKey", "source": "identifier", "name": "ObjectKey" },`
	`702`	`+ { "target": "MultipartUploadId", "source": "identifier", "name": "Id" },`
`703`	`703`	`{ "target": "PartNumber", "source": "input" }`
`704`	`704`	`]`
`705`	`705`	`}`
`@@ -770,8 +770,8 @@`
`770`	`770`	`"type": "MultipartUpload",`
`771`	`771`	`"identifiers": [`
`772`	`772`	`{ "target": "BucketName", "source": "identifier", "name": "BucketName" },`
`773`		`- { "target": "Id", "source": "identifier", "name": "MultipartUploadId" },`
`774`		`- { "target": "ObjectKey", "source": "identifier", "name": "ObjectKey" }`
	`773`	`+ { "target": "ObjectKey", "source": "identifier", "name": "ObjectKey" },`
	`774`	`+ { "target": "Id", "source": "identifier", "name": "MultipartUploadId" }`
`775`	`775`	`]`
`776`	`776`	`}`
`777`	`777`	`}`
Original file line number	Diff line number	Diff line change
`@@ -110,7 +110,6 @@`
`110`	`110`	`"resource": {`
`111`	`111`	`"type": "PlatformEndpoint",`
`112`	`112`	`"identifiers": [`
`113`		`- { "target": "PlatformApplicationArn", "source": "identifier", "name": "Arn" },`
`114`	`113`	`{ "target": "Arn", "source": "response", "path": "EndpointArn" }`
`115`	`114`	`]`
`116`	`115`	`}`