Merge pull request #31 from RachelTucker/update-samples

Updating samples to work with 5.x api

Author: rpmoore

samples/bulk_with_prefix.py

@@ -23,17 +23,17 @@
 fileMap = {}
 
 # this method is used to get the size of the files
-def createDs3Obj(fileName):
+def createDs3PutObject(fileName):
     size = os.stat(fileName).st_size
     ds3ObjName = "prefix/" + fileName
     fileMap[ds3ObjName] = fileName
-    return ds3.FileObject(ds3ObjName, size)
+    return ds3.Ds3PutObject(ds3ObjName, size)
 
 # get the sizes for each file
-fileList = ds3.FileObjectList(list(map(createDs3Obj, fileList)))
+objectList = list(map(createDs3PutObject, fileList))
 
 # submit the put bulk request to DS3
-bulkResult = client.put_bulk_job_spectra_s3(ds3.PutBulkJobSpectraS3Request(bucketName, fileList))
+bulkResult = client.put_bulk_job_spectra_s3(ds3.PutBulkJobSpectraS3Request(bucketName, objectList))
 
 # the bulk request will split the files over several chunks if it needs to
 # we need to iterate over the chunks, ask the server for space to send
@@ -42,10 +42,11 @@ def createDs3Obj(fileName):
     allocateChunk = client.allocate_job_chunk_spectra_s3(ds3.AllocateJobChunkSpectraS3Request(chunk['ChunkId']))
     for obj in allocateChunk.result['ObjectList']:
         objectDataStream = open(fileMap[obj['Name']], "rb")
-        client.put_object(ds3.PutObjectRequest(bucketName, 
-                                               obj['Name'], 
-                                               obj['Length'],
-                                               objectDataStream,
+        objectDataStream.seek(int(obj['Offset']), 0)
+        client.put_object(ds3.PutObjectRequest(bucket_name=bucketName,
+                                               object_name=obj['Name'],
+                                               length=obj['Length'],
+                                               stream=objectDataStream,
                                                offset=int(obj['Offset']), 
                                                job=bulkResult.result['JobId']))
 

samples/getPhysical.py

@@ -16,12 +16,14 @@
 
 client = ds3.createClientFromEnv()
 
-bucketName = "sdkexamples"
-objectName = "123456795.txt"
+# this example assumes that a bucket named "books" and the following object exists on the server
+# note: these are the same objects that are on the server if you run the bulk put example and comment out the delete lines
+bucketName = "books"
+fileList = ["beowulf.txt", "sherlock_holmes.txt", "tale_of_two_cities.txt", "ulysses.txt"]
 
-objlist = ds3.FileObjectList([ds3.FileObject(objectName)]) 
+objectList = list([ds3.Ds3GetObject(name=fileName) for fileName in fileList])
 tapes = client.get_physical_placement_for_objects_spectra_s3(
-               ds3.GetPhysicalPlacementForObjectsSpectraS3Request(bucketName, objlist))
+               ds3.GetPhysicalPlacementForObjectsSpectraS3Request(bucketName, objectList))
 
 for tape in tapes.result['TapeList']:
     print(tape)

samples/gettingData.py

@@ -22,7 +22,7 @@
 
 bucketContents = client.get_bucket(ds3.GetBucketRequest(bucketName))
 
-objectList = ds3.FileObjectList([ds3.FileObject(obj['Key']) for obj in bucketContents.result['ContentsList']])
+objectList = list([ds3.Ds3GetObject(obj['Key']) for obj in bucketContents.result['ContentsList']])
 bulkGetResult = client.get_bulk_job_spectra_s3(ds3.GetBulkJobSpectraS3Request(bucketName, objectList))
 
 # create a set of the chunk ids which will be used to track

samples/puttingData.py

@@ -26,23 +26,21 @@
 
 fileList = ["beowulf.txt", "sherlock_holmes.txt", "tale_of_two_cities.txt", "ulysses.txt"]
 
-# this method is used to get the size of the files
-def getSize(fileName, prefix=""):
-    size = os.stat(pathForResource(fileName)).st_size
-    return ds3.FileObject(prefix + fileName, size)
+# this method is used to map a file path to a Ds3PutObject
+def fileNameToDs3PutObject(filePath, prefix=""):
+    size = os.stat(pathForResource(filePath)).st_size
+    return ds3.Ds3PutObject(prefix + filePath, size)
 
 # this method is used to get the os specific path for an object located in the resources folder
 def pathForResource(resourceName):
-    encoding = sys.getfilesystemencoding()
-    currentPath = os.path.dirname(str(__file__, encoding))
+    currentPath = os.path.dirname(str(__file__))
     return os.path.join(currentPath, "resources", resourceName)
 
 # get the sizes for each file
-fileList = list(map(getSize, fileList))
-fileObjectList = ds3.FileObjectList(fileList)
+fileList = list(map(fileNameToDs3PutObject, fileList))
 
 # submit the put bulk request to DS3
-bulkResult = client.put_bulk_job_spectra_s3(ds3.PutBulkJobSpectraS3Request(bucketName, fileObjectList))
+bulkResult = client.put_bulk_job_spectra_s3(ds3.PutBulkJobSpectraS3Request(bucketName, fileList))
 
 # the bulk request will split the files over several chunks if it needs to.
 # we then need to ask what chunks we can send, and then send them making
@@ -80,6 +78,7 @@ def pathForResource(resourceName):
             if obj['InCache'] == 'false':
                 localFileName = "resources/" + obj['Name']
                 objectDataStream = open(localFileName, "rb")
+                objectDataStream.seek(int(obj['Offset']), 0)
                 client.put_object(ds3.PutObjectRequest(bucketName, 
                                                        obj['Name'], 
                                                        obj['Length'],

samples/renaming.py

@@ -10,7 +10,7 @@
 #   specific language governing permissions and limitations under the License.
 
 import os
-import tempfile
+import time
 
 from ds3 import ds3
 
@@ -36,14 +36,14 @@
     "folder/folder2/ulysses.txt":"resources/ulysses.txt"
 }
 
-# this method is used to get the size of the files
+# this method is used to map a file path to a Ds3PutObject
 # we need two parameters because the S3 API wants the name that the object will take on the server, but the size obviously needs to come from the file on the current file system
-def getSize(fileName, realFileName):
+def fileNameToDs3PutObject(fileName, realFileName):
     size = os.stat(realFileName).st_size
-    return ds3.FileObject(fileName, size)
+    return ds3.Ds3PutObject(fileName, size)
 
 # get the sizes for each file
-fileList = ds3.FileObjectList([getSize(key, fileListMapping[key]) for key in list(fileListMapping.keys())])
+fileList = list([fileNameToDs3PutObject(key, fileListMapping[key]) for key in list(fileListMapping.keys())])
 
 # submit the put bulk request to DS3
 bulkResult = client.put_bulk_job_spectra_s3(ds3.PutBulkJobSpectraS3Request(bucketName, fileList))
@@ -81,17 +81,21 @@ def getSize(fileName, realFileName):
             # it is possible that if we start resending a chunk, due to the program crashing, that 
             # some objects will already be in cache.  Check to make sure that they are not, and then
             # send the object to Spectra S3
-            if not obj['InCache']:
-                client.put_object(PutObjectRequest(bucketName, 
-                                                   obj['Name'], 
-                                                   obj['Offset'], 
-                                                   obj['Length'], 
-                                                   bulkResult.result['JobId'], 
-                                                   real_file_name = fileListMapping[obj.name]))
+            if obj['InCache'] == 'false':
+                objectDataStream = open(fileListMapping[obj['Name']], "rb")
+                objectDataStream.seek(int(obj['Offset']), 0)
+                putObjectResponse = client.put_object(ds3.PutObjectRequest(bucket_name=bucketName,
+                                                   object_name=obj['Name'],
+                                                   offset=obj['Offset'],
+                                                   length=obj['Length'],
+                                                   stream=objectDataStream,
+                                                   job=bulkResult.result['JobId']))
+
 
 # we now verify that all our objects have been sent to DS3
 bucketResponse = client.get_bucket(ds3.GetBucketRequest(bucketName))
 
+print("\nFiles in bucket:")
 for obj in bucketResponse.result['ContentsList']:
     print(obj['Key'])
 
@@ -100,15 +104,15 @@ def getSize(fileName, realFileName):
 
 client.delete_folder_recursively_spectra_s3(ds3.DeleteFolderRecursivelySpectraS3Request(bucketName, "folder/folder2"))
 
-print("\nAfter deletion number 1:")
+print("\nAfter deleting 'folder/folder2':")
 bucketResponse = client.get_bucket(ds3.GetBucketRequest(bucketName))
 
 for obj in bucketResponse.result['ContentsList']:
     print(obj['Key'])
 
 client.delete_folder_recursively_spectra_s3(ds3.DeleteFolderRecursivelySpectraS3Request(bucketName, "folder"))
 
-print("\nAfter deletion number 2:")
+print("\nAfter deleting 'folder':")
 bucketResponse = client.get_bucket(ds3.GetBucketRequest(bucketName))
 
 for obj in bucketResponse.result['ContentsList']: