storage: Add Boto3 List Objects in GCS (GoogleCloudPlatform#2041)

* Add S3 SDK sample to list objects

* Fix test and method names

* Fix typo in helpful information

* Fixing lint issues

* Remove workaround

* Remove debug code from generate_signed_urls.py

* update bucket to blob

Author: frankyn

storage/s3-sdk/README.rst

@@ -79,6 +79,36 @@ To run this sample:
 
 
 
+List GCS Objects using S3 SDK
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+
+.. image:: https://gstatic.com/cloudssh/images/open-btn.png
+   :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=storage/s3-sdk/list_gcs_objects.py,storage/s3-sdk/README.rst
+
+
+
+
+To run this sample:
+
+.. code-block:: bash
+
+    $ python list_gcs_objects.py
+
+    usage: list_gcs_objects.py [-h]
+                               google_access_key_id google_access_key_secret
+                               bucket_name
+
+    positional arguments:
+      google_access_key_id  Your Cloud Storage HMAC Access Key ID.
+      google_access_key_secret
+                            Your Cloud Storage HMAC Access Key Secret.
+      bucket_name           Your Cloud Storage bucket name
+
+    optional arguments:
+      -h, --help            show this help message and exit
+
+
+
 
 
 .. _Google Cloud SDK: https://cloud.google.com/sdk/

storage/s3-sdk/README.rst.in

@@ -17,6 +17,9 @@ samples:
 - name: List GCS Buckets using S3 SDK
   file: list_gcs_buckets.py
   show_help: true
+- name: List GCS Objects using S3 SDK
+  file: list_gcs_objects.py
+  show_help: true
 
 cloud_client_library: false
 

storage/s3-sdk/list_gcs_objects.py

@@ -0,0 +1,60 @@
+#!/usr/bin/env python
+
+# Copyright 2019 Google, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+# [START storage_s3_sdk_list_objects]
+import boto3
+
+
+def list_gcs_objects(google_access_key_id, google_access_key_secret,
+                     bucket_name):
+    """Lists GCS objects using boto3 SDK"""
+    # Create a new client and do the following:
+    # 1. Change the endpoint URL to use the
+    #    Google Cloud Storage XML API endpoint.
+    # 2. Use Cloud Storage HMAC Credentials.
+
+    client = boto3.client("s3", region_name="auto",
+                          endpoint_url="https://storage.googleapis.com",
+                          aws_access_key_id=google_access_key_id,
+                          aws_secret_access_key=google_access_key_secret)
+
+    # Call GCS to list objects in bucket_name
+    response = client.list_objects(Bucket=bucket_name)
+
+    # Print object names
+    print("Objects:")
+    for blob in response["Contents"]:
+        print(blob["Key"])
+# [END storage_s3_sdk_list_objects]
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(
+        description=__doc__,
+        formatter_class=argparse.RawDescriptionHelpFormatter)
+    parser.add_argument("google_access_key_id",
+                        help="Your Cloud Storage HMAC Access Key ID.")
+    parser.add_argument("google_access_key_secret",
+                        help="Your Cloud Storage HMAC Access Key Secret.")
+    parser.add_argument('bucket_name',
+                        help="Your Cloud Storage bucket name")
+
+    args = parser.parse_args()
+
+    list_gcs_objects(google_access_key_id=args.google_access_key_id,
+                     google_access_key_secret=args.google_access_key_secret,
+                     bucket_name=args.bucket_name)

storage/s3-sdk/list_gcs_objects_test.py

@@ -0,0 +1,29 @@
+# Copyright 2019 Google, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+
+import list_gcs_objects
+
+BUCKET = os.environ["GOOGLE_CLOUD_PROJECT_S3_SDK"]
+KEY_ID = os.environ["STORAGE_HMAC_ACCESS_KEY_ID"]
+SECRET_KEY = os.environ["STORAGE_HMAC_ACCESS_SECRET_KEY"]
+
+
+def test_list_blobs(capsys):
+    list_gcs_objects.list_gcs_objects(google_access_key_id=KEY_ID,
+                                      google_access_key_secret=SECRET_KEY,
+                                      bucket_name=BUCKET)
+    out, _ = capsys.readouterr()
+    assert "Objects:" in out