Refactor object storage API test script

Turn the single monolithic script into a separate module for each
provder, an abstract base class for all of the providers, and a test
file which now only does workload generation and benchmarking.

perfkitbenchmarker/linux_benchmarks/object_storage_service_benchmark.py

@@ -177,6 +177,7 @@
 LARGE_DATA_SIZE_IN_MBITS = 8 * LARGE_DATA_SIZE_IN_BYTES / 1000 / 1000
 
 API_TEST_SCRIPT = 'object_storage_api_tests.py'
+API_TEST_SCRIPTS_DIR = 'object_storage_api_test_scripts'
 
 # Various constants to name the result metrics.
 THROUGHPUT_UNIT = 'Mbps'
@@ -1440,8 +1441,14 @@ def Prepare(benchmark_spec):
   file_path = data.ResourcePath(DATA_FILE)
   vms[0].PushFile(file_path, '%s/run/' % scratch_dir)
 
-  api_test_script_path = data.ResourcePath(API_TEST_SCRIPT)
-  vms[0].PushFile(api_test_script_path, '%s/run/' % scratch_dir)
+  api_test_scripts_path = data.ResourcePath(API_TEST_SCRIPTS_DIR)
+  logging.info('api_test_scripts_path: %s, %s',
+               API_TEST_SCRIPTS_DIR,
+               api_test_scripts_path)
+  for path in os.listdir(api_test_scripts_path):
+    logging.info('Uploading %s', path)
+    vms[0].PushFile('%s/%s' % (api_test_scripts_path, path),
+                    '%s/run/' % scratch_dir)
 
 
 def Run(benchmark_spec):

perfkitbenchmarker/scripts/object_storage_api_test_scripts/azure_service.py

@@ -0,0 +1,67 @@
+# Copyright 2016 PerfKitBenchmarker Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""An interface to the Azure Blob Storage API."""
+
+import logging
+import time
+
+import gflags as flags
+import azure.storage.blob
+
+import object_storage_interface
+
+FLAGS = flags.FLAGS
+
+flags.DEFINE_string('azure_account', None,
+                    'The name of the storage account for Azure.')
+
+flags.DEFINE_string('azure_key', None,
+                    'The key of the storage account for Azure.')
+
+
+class AzureService(object_storage_interface.ObjectStorageServiceBase):
+  def __init__(self):
+    if FLAGS.azure_key is None or FLAGS.azure_account is None:
+      raise ValueError('Must specify azure account and key.')
+    self.blobService = azure.storage.blob.BlobService(FLAGS.azure_account,
+                                                      FLAGS.azure_key)
+
+  def ListObjects(self, bucket, prefix):
+    return [obj.name
+            for obj in self.blobService.list_blobs(bucket, prefix=prefix)]
+
+  def DeleteObjects(self, bucket, objects_to_delete, objects_deleted=None):
+    for object_name in objects_to_delete:
+      try:
+        self.blobService.delete_blob(bucket, object_name)
+        if objects_deleted is not None:
+          objects_deleted.append(object_name)
+      except:
+        logging.exception('Caught exception while deleting object %s.',
+                          object_name)
+
+  def WriteObjectFromBuffer(self, bucket, object, stream, size):
+    stream.seek(0)
+    start_time = time.time()
+    self.blobService.put_block_blob_from_file(
+        bucket, object, stream, count=size)
+    latency = time.time() - start_time
+    return start_time, latency
+
+  def ReadObject(self, bucket, object):
+    start_time = time.time()
+    self.blobService.get_blob_to_bytes(bucket, object)
+    latency = time.time() - start_time
+    return start_time, latency

perfkitbenchmarker/scripts/object_storage_api_test_scripts/boto_service.py

@@ -0,0 +1,74 @@
+# Copyright 2016 PerfKitBenchmarker Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""An interface to boto-based object storage APIs."""
+
+import logging
+import time
+
+import boto
+
+import object_storage_interface
+
+
+class BotoService(object_storage_interface.ObjectStorageServiceBase):
+  def __init__(self, storage_schema, host_to_connect=None):
+    self.storage_schema = storage_schema
+    self.host_to_connect = host_to_connect
+
+  def _StorageURI(self, bucket, object=None):
+    """Return a storage_uri for the given resource.
+
+    Args:
+      bucket: the name of a bucket.
+      object: the name of an object, if given.
+
+    Returns: a storage_uri. If object is given, the uri will be for
+    the bucket-object combination. If object is not given, the uri
+    will be for the bucket.
+    """
+
+    if object is not None:
+      path = '%s/%s' % (bucket, object)
+    else:
+      path = bucket
+    storage_uri = boto.storage_uri(path, self.storage_schema)
+    if self.host_to_connect is not None:
+      storage_uri.connect(host=self.host_to_connect)
+    return storage_uri
+
+  def ListObjects(self, bucket, prefix):
+    bucket_uri = self._StorageURI(bucket)
+    return [obj.name for obj in bucket_uri.list_bucket(prefix=prefix)]
+
+  def DeleteObjects(self, bucket, objects_to_delete, objects_deleted=None):
+    for object_name in objects_to_delete:
+      try:
+        object_uri = self._StorageURI(bucket, object_name)
+        object_uri.delete_key()
+        if objects_deleted is not None:
+          objects_deleted.append(object_name)
+      except:
+        logging.exception('Caught exception while deleting object %s.',
+                          object_name)
+
+  # Not implementing WriteObjectFromBuffer because the implementation
+  # is different for GCS and S3.
+
+  def ReadObject(self, bucket, object):
+    start_time = time.time()
+    object_uri = self._StorageURI(bucket, object)
+    object_uri.new_key().get_contents_as_string()
+    latency = time.time() - start_time
+    return start_time, latency

perfkitbenchmarker/scripts/object_storage_api_test_scripts/gcs.py

@@ -0,0 +1,40 @@
+# Copyright 2016 PerfKitBenchmarker Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""An interface to Google Cloud Storage, using the boto library."""
+
+import logging
+import time
+
+import gflags as flags
+
+import boto_service
+
+FLAGS = flags.FLAGS
+
+
+class GCSService(boto_service.BotoService):
+  def __init__(self):
+    # GCS does not use separate endpoints, so host_to_connect makes no sense.
+    if FLAGS.host is not None:
+      logging.warning('GCSService ignoring --host=%s', FLAGS.host)
+    super(GCSService, self).__init__('gs', host_to_connect=None)
+
+  def WriteObjectFromBuffer(self, bucket, object, stream, size):
+    stream.seek(0)
+    start_time = time.time()
+    object_uri = self._StorageURI(bucket, object)
+    object_uri.set_contents_from_file(stream, size=size)
+    latency = time.time() - start_time
+    return start_time, latency