Skip to content

Commit f8e61b9

Browse files
mahendra-googlemahendra-google
authored
samples(storage transfer): add samples and test cases for storage transfer services (#2857)
1 parent 104782c commit f8e61b9

19 files changed

+1153
-88
lines changed

‎storagetransfer/api/StorageTransfer.Samples.Tests/CheckLatestTransferOperationTest.cs‎

Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,92 @@
1+
// Copyright 2024 Google LLC
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License").
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// https://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
using Google.Cloud.StorageTransfer.V1;
16+
using System;
17+
using Xunit;
18+
19+
namespace StorageTransfer.Samples.Tests;
20+
21+
[Collection(nameof(StorageFixture))]
22+
public class CheckLatestTransferOperationTest : IDisposable
23+
{
24+
private readonly StorageFixture _fixture;
25+
private string _jobName;
26+
private readonly string _transferJobName;
27+
private readonly string _sourceBucket;
28+
private readonly string _sinkBucket;
29+
30+
public CheckLatestTransferOperationTest(StorageFixture fixture)
31+
{
32+
_fixture = fixture;
33+
_sourceBucket = _fixture.GenerateBucketName();
34+
_sinkBucket = _fixture.GenerateBucketName();
35+
_fixture.CreateBucketAndGrantStsPermissions(_sourceBucket);
36+
_fixture.CreateBucketAndGrantStsPermissions(_sinkBucket);
37+
_transferJobName = CreateTransferJob();
38+
}
39+
40+
[Fact]
41+
public void CheckLatestTransferOperation()
42+
{
43+
CheckLatestTransferOperationSample checkLatestTransferOperationSample = new CheckLatestTransferOperationSample();
44+
var transferJob = checkLatestTransferOperationSample.CheckLatestTransferOperation(_fixture.ProjectId, _transferJobName);
45+
Assert.Contains("transferJobs/", transferJob.Name);
46+
_jobName = transferJob.Name;
47+
}
48+
49+
private string CreateTransferJob()
50+
{
51+
TransferJob transferJob = new TransferJob
52+
{
53+
ProjectId = _fixture.ProjectId,
54+
TransferSpec = new TransferSpec
55+
{
56+
GcsDataSink = new GcsData { BucketName = _sinkBucket },
57+
GcsDataSource = new GcsData { BucketName = _sourceBucket }
58+
},
59+
Status = TransferJob.Types.Status.Enabled
60+
};
61+
CreateTransferJobRequest request = new CreateTransferJobRequest
62+
{
63+
TransferJob = transferJob
64+
};
65+
66+
TransferJob response = _fixture.Sts.CreateTransferJob(new CreateTransferJobRequest { TransferJob = transferJob });
67+
return response.Name;
68+
}
69+
70+
public void Dispose()
71+
{
72+
try
73+
{
74+
_fixture.Sts.UpdateTransferJob(new UpdateTransferJobRequest()
75+
{
76+
ProjectId = _fixture.ProjectId,
77+
JobName = _transferJobName,
78+
TransferJob = new TransferJob()
79+
{
80+
Name = _transferJobName,
81+
Status = TransferJob.Types.Status.Deleted
82+
}
83+
});
84+
_fixture.Storage.DeleteBucket(_sourceBucket);
85+
_fixture.Storage.DeleteBucket(_sinkBucket);
86+
}
87+
catch (Exception)
88+
{
89+
// Do nothing, we delete on a best effort basis.
90+
}
91+
}
92+
}

‎storagetransfer/api/StorageTransfer.Samples.Tests/CreateEventDrivenGcsTransferTest.cs‎

Lines changed: 114 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,114 @@
1+
/**
2+
* Copyright 2024 Google Inc.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
using Google.Cloud.PubSub.V1;
18+
using Google.Cloud.StorageTransfer.V1;
19+
using System;
20+
using Xunit;
21+
22+
namespace StorageTransfer.Samples.Tests;
23+
24+
[Collection(nameof(StorageFixture))]
25+
public class CreateEventDrivenGcsTransferTest : IDisposable
26+
{
27+
private readonly StorageFixture _fixture;
28+
private readonly string _pubSubId;
29+
private string _transferJobName;
30+
private readonly string _sourceBucket;
31+
private readonly string _sinkBucket;
32+
private string TopicId { get; } = $"Topic-{Guid.NewGuid().ToString()}";
33+
private string SubscriptionId { get; } = $"Subscription-{Guid.NewGuid().ToString()}";
34+
private SubscriberServiceApiClient SubscriberClient { get; } = SubscriberServiceApiClient.Create();
35+
private PublisherServiceApiClient PublisherClient { get; } = PublisherServiceApiClient.Create();
36+
37+
public CreateEventDrivenGcsTransferTest(StorageFixture fixture)
38+
{
39+
_fixture = fixture;
40+
_sourceBucket = _fixture.GenerateBucketName();
41+
_sinkBucket = _fixture.GenerateBucketName();
42+
_fixture.CreateBucketAndGrantStsPermissions(_sourceBucket);
43+
_fixture.CreateBucketAndGrantStsPermissions(_sinkBucket);
44+
_pubSubId = $"projects/{_fixture.ProjectId}/subscriptions/{SubscriptionId}";
45+
CreatePubSubResourcesAndGrantStsPermissions();
46+
}
47+
48+
[Fact]
49+
public void CreateEventDrivenGcsTransfer()
50+
{
51+
CreateEventDrivenGcsTransferSample createEventDrivenGcsTransferSample = new CreateEventDrivenGcsTransferSample();
52+
var transferJob = createEventDrivenGcsTransferSample.CreateEventDrivenGcsTransfer(_fixture.ProjectId, _sourceBucket, _sinkBucket, _pubSubId);
53+
Assert.Contains("transferJobs/", transferJob.Name);
54+
_transferJobName = transferJob.Name;
55+
}
56+
57+
private void CreatePubSubResourcesAndGrantStsPermissions()
58+
{
59+
string email = _fixture.Sts.GetGoogleServiceAccount(new GetGoogleServiceAccountRequest()
60+
{
61+
ProjectId = _fixture.ProjectId
62+
}).AccountEmail;
63+
64+
string memberServiceAccount = "serviceAccount:" + email;
65+
SubscriptionName subscriptionName = new SubscriptionName(_fixture.ProjectId, SubscriptionId);
66+
TopicName topicName = new TopicName(_fixture.ProjectId, TopicId);
67+
PublisherClient.CreateTopic(topicName);
68+
SubscriberClient.CreateSubscription(subscriptionName, topicName, pushConfig: null, ackDeadlineSeconds: 500);
69+
70+
var policyIamPolicyTopic = new Google.Cloud.Iam.V1.Policy();
71+
policyIamPolicyTopic.AddRoleMember("roles/pubsub.publisher", memberServiceAccount);
72+
PublisherClient.IAMPolicyClient.SetIamPolicy(new Google.Cloud.Iam.V1.SetIamPolicyRequest
73+
{
74+
ResourceAsResourceName = topicName,
75+
Policy = policyIamPolicyTopic
76+
});
77+
78+
var policyIamPolicySubscriber = new Google.Cloud.Iam.V1.Policy();
79+
policyIamPolicySubscriber.AddRoleMember("roles/pubsub.subscriber", memberServiceAccount);
80+
PublisherClient.IAMPolicyClient.SetIamPolicy(new Google.Cloud.Iam.V1.SetIamPolicyRequest
81+
{
82+
ResourceAsResourceName = subscriptionName,
83+
Policy = policyIamPolicySubscriber
84+
});
85+
}
86+
87+
public void Dispose()
88+
{
89+
try
90+
{
91+
_fixture.Sts.UpdateTransferJob(new UpdateTransferJobRequest()
92+
{
93+
ProjectId = _fixture.ProjectId,
94+
JobName = _transferJobName,
95+
TransferJob = new TransferJob()
96+
{
97+
Name = _transferJobName,
98+
Status = TransferJob.Types.Status.Deleted
99+
}
100+
});
101+
102+
TopicName topicName = TopicName.FromProjectTopic(_fixture.ProjectId, TopicId);
103+
PublisherClient.DeleteTopic(topicName);
104+
SubscriptionName subscriptionName = SubscriptionName.FromProjectSubscription(_fixture.ProjectId, SubscriptionId);
105+
SubscriberClient.DeleteSubscription(subscriptionName);
106+
_fixture.Storage.DeleteBucket(_sourceBucket);
107+
_fixture.Storage.DeleteBucket(_sinkBucket);
108+
}
109+
catch (Exception)
110+
{
111+
// Do nothing, we delete on a best effort basis.
112+
}
113+
}
114+
}

‎storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs‎

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
// Copyright 2024 Google LLC
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License").
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// https://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
using Google.Cloud.Storage.V1;
16+
using Google.Cloud.StorageTransfer.V1;
17+
using System;
18+
using System.IO;
19+
using Xunit;
20+
21+
namespace StorageTransfer.Samples.Tests;
22+
23+
[Collection(nameof(StorageFixture))]
24+
public class DownloadToPosixTest : IDisposable
25+
{
26+
private readonly StorageFixture _fixture;
27+
private string _transferJobName;
28+
private readonly string _tempDirectory;
29+
private readonly string _gcsSourcePath;
30+
private readonly string _sourceBucket;
31+
32+
public DownloadToPosixTest(StorageFixture fixture)
33+
{
34+
_fixture = fixture;
35+
_sourceBucket = _fixture.GenerateBucketName();
36+
_fixture.CreateBucketAndGrantStsPermissions(_sourceBucket);
37+
_tempDirectory = _fixture.GenerateTempFolderPath();
38+
_gcsSourcePath = $"{Guid.NewGuid()}/{Guid.NewGuid()}/";
39+
UploadObjectToBucket(_sourceBucket);
40+
}
41+
42+
[Fact]
43+
public void DownloadToPosix()
44+
{
45+
DownloadToPosixSample downloadToPosixSample = new DownloadToPosixSample();
46+
Directory.CreateDirectory(_tempDirectory);
47+
var transferJob = downloadToPosixSample.DownloadToPosix(_fixture.ProjectId, _fixture.SinkAgentPoolName, _sourceBucket, _gcsSourcePath, _tempDirectory);
48+
Assert.Contains("transferJobs/", transferJob.Name);
49+
Assert.True(Directory.Exists(_tempDirectory));
50+
_transferJobName = transferJob.Name;
51+
}
52+
53+
private void UploadObjectToBucket(string bucketName)
54+
{
55+
byte[] byteArray = System.Text.Encoding.UTF8.GetBytes($@"{Guid.NewGuid()}.jpeg");
56+
MemoryStream stream = new MemoryStream(byteArray);
57+
string fileName = $"{_gcsSourcePath}{DateTime.Now.ToString("yyyyMMddHHmmss")}.txt";
58+
_fixture.Storage.UploadObject(bucketName, fileName, "application/octet-stream", stream);
59+
}
60+
61+
public void Dispose()
62+
{
63+
try
64+
{
65+
_fixture.Sts.UpdateTransferJob(new UpdateTransferJobRequest()
66+
{
67+
ProjectId = _fixture.ProjectId,
68+
JobName = _transferJobName,
69+
TransferJob = new TransferJob()
70+
{
71+
Name = _transferJobName,
72+
Status = TransferJob.Types.Status.Deleted
73+
}
74+
});
75+
Directory.Delete(_tempDirectory, true);
76+
_fixture.Storage.DeleteBucket(_sourceBucket, new DeleteBucketOptions { DeleteObjects = true });
77+
}
78+
catch (Exception)
79+
{
80+
// Do nothing, we delete on a best effort basis.
81+
}
82+
}
83+
}

storagetransfer/api/StorageTransfer.Samples.Tests/QuickstartTest.cs renamed to storagetransfer/api/StorageTransfer.Samples.Tests/QuickStartTest.cs

Lines changed: 15 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
/**
1+
/**
22
* Copyright 2021 Google Inc.
33
*
44
* Licensed under the Apache License, Version 2.0 (the "License");
@@ -14,28 +14,34 @@
1414
* limitations under the License.
1515
*/
1616

17-
using System;
1817
using Google.Cloud.StorageTransfer.V1;
18+
using System;
1919
using Xunit;
2020

2121
namespace StorageTransfer.Samples.Tests
2222
{
2323
[Collection(nameof(StorageFixture))]
24-
public class QuickstartTest : IDisposable
24+
public class QuickStartTest : IDisposable
2525
{
2626
private readonly StorageFixture _fixture;
2727
private string _transferJobName;
28+
private readonly string _sourceBucket;
29+
private readonly string _sinkBucket;
2830

29-
public QuickstartTest(StorageFixture fixture)
31+
public QuickStartTest(StorageFixture fixture)
3032
{
3133
_fixture = fixture;
34+
_sourceBucket = _fixture.GenerateBucketName();
35+
_sinkBucket = _fixture.GenerateBucketName();
36+
_fixture.CreateBucketAndGrantStsPermissions(_sourceBucket);
37+
_fixture.CreateBucketAndGrantStsPermissions(_sinkBucket);
3238
}
3339

3440
[Fact]
35-
public void TestQuickstart()
41+
public void QuickStart()
3642
{
37-
QuickstartSample quickstartSample = new QuickstartSample();
38-
var transferJob = quickstartSample.Quickstart(_fixture.ProjectId, _fixture.BucketNameSource, _fixture.BucketNameSink);
43+
QuickStartSample quickStartSample = new QuickStartSample();
44+
var transferJob = quickStartSample.QuickStart(_fixture.ProjectId, _sourceBucket, _sinkBucket);
3945
Assert.Contains("transferJobs/", transferJob.Name);
4046
_transferJobName = transferJob.Name;
4147
}
@@ -54,6 +60,8 @@ public void Dispose()
5460
Status = TransferJob.Types.Status.Deleted
5561
}
5662
});
63+
_fixture.Storage.DeleteBucket(_sourceBucket);
64+
_fixture.Storage.DeleteBucket(_sinkBucket);
5765
}
5866
catch (Exception)
5967
{

0 commit comments

Comments
 (0)