Skip to content

Commit b20d8cb

Browse files
rpmoorerpmoore
authored
Merge pull request #40 from RachelTucker/GOSDK-3
GOSDK-3: Added samples for get service, bulk put, and bulk get
2 parents 4094d9d + d532e60 commit b20d8cb

File tree

8 files changed

+69696
-0
lines changed

8 files changed

+69696
-0
lines changed

‎src/samples/getBulkSample.go‎

Lines changed: 99 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,99 @@
1+
// Copyright 2014-2017 Spectra Logic Corporation. All Rights Reserved.
2+
// Licensed under the Apache License, Version 2.0 (the "License"). You may not use
3+
// this file except in compliance with the License. A copy of the License is located at
4+
//
5+
// http://www.apache.org/licenses/LICENSE-2.0
6+
//
7+
// or in the "license" file accompanying this file.
8+
// This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
9+
// CONDITIONS OF ANY KIND, either express or implied. See the License for the
10+
// specific language governing permissions and limitations under the License.
11+
12+
package main
13+
14+
import (
15+
"ds3/buildclient"
16+
"log"
17+
"ds3/models"
18+
"samples/utils"
19+
"time"
20+
)
21+
22+
// Demonstrates how to perform a bulk get. Assumes that the target bucket already exists
23+
// and has files, i.e. run putBulkSample.go first.
24+
func main() {
25+
// Create a client from environment variables.
26+
client, err := buildclient.FromEnv()
27+
if err != nil {
28+
log.Fatal(err)
29+
}
30+
31+
// Get a list of all objects in the target bucket.
32+
bucketResponse, err := client.GetBucket(models.NewGetBucketRequest(utils.BucketName))
33+
if err != nil {
34+
log.Fatal(err)
35+
}
36+
37+
objectList := bucketResponse.ListBucketResult.Objects
38+
39+
// Convert object list returned from get bucket into a list of object names.
40+
var objectNames []string
41+
for _, obj := range objectList {
42+
objectNames = append(objectNames, *obj.Key)
43+
}
44+
45+
// Initialize the bulk get request.
46+
bulkGetRequest := models.NewGetBulkJobSpectraS3Request(utils.BucketName, objectNames)
47+
48+
// Send the bulk get request to the server. Note that this creates the bulk get job,
49+
// but does not retrieve the objects.
50+
bulkGetResponse, err := client.GetBulkJobSpectraS3(bulkGetRequest)
51+
if err != nil {
52+
log.Fatal(err)
53+
}
54+
55+
// Bulk jobs are split into multiple chunks which then need to be transferred.
56+
totalChunkCount := len(bulkGetResponse.MasterObjectList.Objects)
57+
curChunkCount := 0
58+
59+
for curChunkCount < totalChunkCount {
60+
// Get the chunks that the server can send. The server may need to retrieve
61+
// objects into cache from the tape.
62+
chunksReady := models.NewGetJobChunksReadyForClientProcessingSpectraS3Request(bulkGetResponse.MasterObjectList.JobId)
63+
chunksReadyResponse, err := client.GetJobChunksReadyForClientProcessingSpectraS3(chunksReady)
64+
if err != nil {
65+
log.Fatal(err)
66+
}
67+
68+
// Check to see if any chunks can be processed.
69+
numberOfChunks := len(chunksReadyResponse.MasterObjectList.Objects)
70+
if numberOfChunks > 0 {
71+
// Loop through all the chunks that are available for processing, and get
72+
// the files that are contained within them.
73+
for _, curChunk := range chunksReadyResponse.MasterObjectList.Objects {
74+
75+
for _, curObj := range curChunk.Objects {
76+
77+
getObjRequest := models.NewGetObjectRequest(utils.BucketName, *curObj.Name).
78+
WithJob(bulkGetResponse.MasterObjectList.JobId).
79+
WithOffset(curObj.Offset)
80+
81+
getObjResponse, err := client.GetObject(getObjRequest)
82+
if err != nil {
83+
log.Fatal(err)
84+
}
85+
86+
err = utils.VerifyBookContent(*curObj.Name, getObjResponse.Content)
87+
if err != nil {
88+
log.Fatal(err)
89+
}
90+
}
91+
curChunkCount++
92+
}
93+
} else {
94+
// When no chunks are returned we need to sleep to allow for cache space to
95+
// be freed.
96+
time.Sleep(time.Second * 5)
97+
}
98+
}
99+
}

‎src/samples/getServiceSample.go‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
// Copyright 2014-2017 Spectra Logic Corporation. All Rights Reserved.
2+
// Licensed under the Apache License, Version 2.0 (the "License"). You may not use
3+
// this file except in compliance with the License. A copy of the License is located at
4+
//
5+
// http://www.apache.org/licenses/LICENSE-2.0
6+
//
7+
// or in the "license" file accompanying this file.
8+
// This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
9+
// CONDITIONS OF ANY KIND, either express or implied. See the License for the
10+
// specific language governing permissions and limitations under the License.
11+
12+
package main
13+
14+
import (
15+
"log"
16+
"ds3/models"
17+
"ds3/buildclient"
18+
"fmt"
19+
"samples/utils"
20+
)
21+
22+
func main() {
23+
// Create the client from environment variables.
24+
client, err := buildclient.FromEnv()
25+
if err != nil {
26+
log.Fatal(err)
27+
}
28+
29+
// Create the get service request. All requests to a DS3 appliance start this way.
30+
request := models.NewGetServiceRequest()
31+
32+
// Perform the Get Service call by using the client and invoking the desired command.
33+
response, err := client.GetService(request)
34+
if err != nil {
35+
log.Fatal(err)
36+
}
37+
38+
// Printing contents of get service.
39+
fmt.Println("Buckets:")
40+
for i, bucket := range response.ListAllMyBucketsResult.Buckets {
41+
fmt.Printf("%d) Name: %s; CreationDate: %s\n", i + 1, utils.ToSafeString(bucket.Name), utils.ToSafeString(bucket.CreationDate))
42+
}
43+
}

‎src/samples/putBulkSample.go‎

Lines changed: 107 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,107 @@
1+
// Copyright 2014-2017 Spectra Logic Corporation. All Rights Reserved.
2+
// Licensed under the Apache License, Version 2.0 (the "License"). You may not use
3+
// this file except in compliance with the License. A copy of the License is located at
4+
//
5+
// http://www.apache.org/licenses/LICENSE-2.0
6+
//
7+
// or in the "license" file accompanying this file.
8+
// This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
9+
// CONDITIONS OF ANY KIND, either express or implied. See the License for the
10+
// specific language governing permissions and limitations under the License.
11+
12+
package main
13+
14+
import (
15+
"log"
16+
"ds3/models"
17+
"ds3/buildclient"
18+
"os"
19+
"time"
20+
"samples/utils"
21+
)
22+
23+
func main() {
24+
// Create a client from environment variables.
25+
client, err := buildclient.FromEnv()
26+
if err != nil {
27+
log.Fatal(err)
28+
}
29+
30+
// Create a bucket where our files will be stored.
31+
_, err = client.PutBucket(models.NewPutBucketRequest(utils.BucketName))
32+
if err != nil {
33+
log.Fatal(err)
34+
}
35+
36+
// Create the list of Ds3PutObjects to be put to the Black Pearl via the bulk put
37+
// command. For each object we need its name and size.
38+
var ds3PutObjects []models.Ds3PutObject
39+
for _, book := range utils.BookNames {
40+
fi, err := os.Stat(utils.ResourceFolder + book)
41+
if err != nil {
42+
log.Fatal(err)
43+
}
44+
curObj := models.Ds3PutObject{ Name:book, Size:fi.Size()}
45+
ds3PutObjects = append(ds3PutObjects, curObj)
46+
}
47+
48+
// Create the bulk put request.
49+
putBulkRequest := models.NewPutBulkJobSpectraS3Request(utils.BucketName, ds3PutObjects)
50+
51+
// Send the bulk put request to the server. This creates the job, but does not
52+
// directly send the data.
53+
putBulkResponse, err := client.PutBulkJobSpectraS3(putBulkRequest)
54+
if err != nil {
55+
log.Fatal(err)
56+
}
57+
58+
// The bulk request will split the files over several chunks if it needs to.
59+
// We then need to ask what chunks we can send, and then send them making sure
60+
// we don't resend the same chunks.
61+
62+
// The bulk job is split into multiple chunks, which then need to be transferred.
63+
totalChunkCount := len(putBulkResponse.MasterObjectList.Objects)
64+
curChunkCount := 0
65+
66+
for curChunkCount < totalChunkCount {
67+
// Get the list of available chunks that the server can receive. The server may
68+
// not be able to receive everything, so not all chunks will necessarily be
69+
// returned
70+
chunksReady := models.NewGetJobChunksReadyForClientProcessingSpectraS3Request(putBulkResponse.MasterObjectList.JobId)
71+
chunksReadyResponse, err := client.GetJobChunksReadyForClientProcessingSpectraS3(chunksReady)
72+
if err != nil {
73+
log.Fatal(err)
74+
}
75+
76+
// Check to see if any chunks can be processed
77+
numberOfChunks := len(chunksReadyResponse.MasterObjectList.Objects)
78+
if numberOfChunks > 0 {
79+
// Loop through all the chunks that are available for processing, and send
80+
// the files that are contained within them.
81+
for _, curChunk := range chunksReadyResponse.MasterObjectList.Objects {
82+
83+
for _, curObj := range curChunk.Objects {
84+
85+
reader, err := utils.LoadBook(*curObj.Name)
86+
if err != nil {
87+
log.Fatal(err)
88+
}
89+
90+
putObjRequest := models.NewPutObjectRequest(utils.BucketName, *curObj.Name, *reader).
91+
WithJob(chunksReadyResponse.MasterObjectList.JobId).
92+
WithOffset(curObj.Offset)
93+
94+
_, err = client.PutObject(putObjRequest)
95+
if err != nil {
96+
log.Fatal(err)
97+
}
98+
}
99+
curChunkCount++
100+
}
101+
} else {
102+
// When no chunks are returned we need to sleep to allow for cache space to
103+
// be freed.
104+
time.Sleep(time.Second * 5)
105+
}
106+
}
107+
}

0 commit comments

Comments
 (0)