Skip to content

Commit cab29ce

Browse files
rpmoorerpmoore
authored
Merge pull request #484 from RachelTucker/python-get-object-buffer
Updating python generation to support settable buffer size in get object
2 parents bd1f9e5 + 4a35401 commit cab29ce

File tree

15 files changed

+268
-27
lines changed

15 files changed

+268
-27
lines changed

‎ds3-autogen-python/src/main/java/com/spectralogic/ds3autogen/python/PythonCodeGenerator.java‎

Lines changed: 26 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
import com.spectralogic.ds3autogen.api.models.docspec.Ds3DocSpec;
2626
import com.spectralogic.ds3autogen.python.generators.client.BaseClientGenerator;
2727
import com.spectralogic.ds3autogen.python.generators.client.ClientModelGenerator;
28+
import com.spectralogic.ds3autogen.python.generators.client.GetObjectCommandGenerator;
2829
import com.spectralogic.ds3autogen.python.generators.request.*;
2930
import com.spectralogic.ds3autogen.python.generators.response.*;
3031
import com.spectralogic.ds3autogen.python.generators.type.BaseTypeGenerator;
@@ -38,6 +39,7 @@
3839
import com.spectralogic.ds3autogen.utils.Helper;
3940
import com.spectralogic.ds3autogen.utils.collections.GuavaCollectors;
4041
import freemarker.template.*;
42+
import org.jetbrains.annotations.NotNull;
4143
import org.slf4j.Logger;
4244
import org.slf4j.LoggerFactory;
4345

@@ -146,10 +148,20 @@ protected static ImmutableList<BaseClient> toClientCommands(
146148
* Generates the python model for a client command
147149
*/
148150
protected static BaseClient toClientCommand(final Ds3Request ds3Request, final Ds3DocSpec docSpec) {
149-
final ClientModelGenerator<?> clientGenerator = new BaseClientGenerator();
151+
final ClientModelGenerator<?> clientGenerator = getClientGenerator(ds3Request);
150152
return clientGenerator.generate(ds3Request, docSpec);
151153
}
152154

155+
/**
156+
* Retrieves the appropriate generator for creating the specified client command
157+
*/
158+
static ClientModelGenerator<?> getClientGenerator(final Ds3Request ds3Request) {
159+
if (isGetObjectAmazonS3Request(ds3Request)) {
160+
return new GetObjectCommandGenerator();
161+
}
162+
return new BaseClientGenerator();
163+
}
164+
153165
/**
154166
* Generates the python models for type descriptors, which are used to describe expected responses
155167
*/
@@ -257,33 +269,33 @@ public BaseRequestGenerator getPutObjectRequestGenerator() {
257269
/**
258270
* Converts all Ds3Requests into the python response handler models
259271
*/
260-
protected static ImmutableList<BaseResponse> toResponseModelList(
272+
protected ImmutableList<BaseResponse> toResponseModelList(
261273
final ImmutableList<Ds3Request> ds3Requests) {
262274
if (isEmpty(ds3Requests)) {
263275
return ImmutableList.of();
264276
}
265277
return ds3Requests.stream()
266-
.map(PythonCodeGenerator::toResponseModel)
278+
.map(this::toResponseModel)
267279
.collect(GuavaCollectors.immutableList());
268280
}
269281

270282
/**
271283
* Converts a Ds3Request into a python response handler model
272284
*/
273-
protected static BaseResponse toResponseModel(final Ds3Request ds3Request) {
285+
protected BaseResponse toResponseModel(final Ds3Request ds3Request) {
274286
final ResponseModelGenerator<?> responseGenerator = getResponseGenerator(ds3Request);
275287
return responseGenerator.generate(ds3Request);
276288
}
277289

278290
/**
279291
* Retrieves the Response Generator associated with the Ds3Request
280292
*/
281-
protected static ResponseModelGenerator<?> getResponseGenerator(final Ds3Request ds3Request) {
293+
protected ResponseModelGenerator<?> getResponseGenerator(final Ds3Request ds3Request) {
282294
if (supportsPaginationRequest(ds3Request)) {
283295
return new PaginationResponseGenerator();
284296
}
285297
if (isGetObjectAmazonS3Request(ds3Request)) {
286-
return new GetObjectResponseGenerator();
298+
return getGetObjectResponseGenerator();
287299
}
288300
if (isHeadObjectRequest(ds3Request)) {
289301
return new HeadObjectResponseGenerator();
@@ -293,4 +305,12 @@ protected static ResponseModelGenerator<?> getResponseGenerator(final Ds3Request
293305
}
294306
return new BaseResponseGenerator();
295307
}
308+
309+
/**
310+
* Retrieves the Python 2.7 compatible generator for the GetObjectResponse
311+
*/
312+
@Override
313+
public BaseResponseGenerator getGetObjectResponseGenerator() {
314+
return new GetObjectResponseGenerator();
315+
}
296316
}

‎ds3-autogen-python/src/main/java/com/spectralogic/ds3autogen/python/generators/client/BaseClientGenerator.java‎

Lines changed: 16 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -25,31 +25,43 @@
2525
import static com.spectralogic.ds3autogen.utils.NormalizingContractNamesUtil.removePath;
2626
import static com.spectralogic.ds3autogen.utils.NormalizingContractNamesUtil.toResponseName;
2727

28-
public class BaseClientGenerator implements ClientModelGenerator<BaseClient> {
28+
public class BaseClientGenerator implements ClientModelGenerator<BaseClient>, ClientModelGeneratorUtils {
2929

3030
@Override
3131
public BaseClient generate(final Ds3Request ds3Request, final Ds3DocSpec docSpec) {
3232
final String commandName = toPythonCommandName(ds3Request.getName());
3333
final String requestType = removePath(ds3Request.getName());
3434
final String responseName = toResponseName(ds3Request.getName());
3535
final String documentation = toDocumentation(ds3Request.getName(), docSpec);
36+
final String funcParams = getFunctionParameters();
37+
final String responseParams = getResponseParameters();
3638

37-
return new BaseClient(commandName, requestType, responseName, documentation);
39+
return new BaseClient(commandName, requestType, responseName, documentation, funcParams, responseParams);
3840
}
3941

4042
/**
4143
* Creates the client documentation for the request
4244
* @param requestName The request name with path
4345
*/
44-
protected static String toDocumentation(final String requestName, final Ds3DocSpec docSpec) {
46+
static String toDocumentation(final String requestName, final Ds3DocSpec docSpec) {
4547
return toCommandDocs(removePath(requestName), docSpec, 1);
4648
}
4749

4850
/**
4951
* Converts the Ds3Request name into the command name used in the client
5052
* ex: GetBucketRequest -> get_bucket
5153
*/
52-
protected static String toPythonCommandName(final String requestName) {
54+
static String toPythonCommandName(final String requestName) {
5355
return camelToUnderscore(toCommandName(requestName));
5456
}
57+
58+
@Override
59+
public String getFunctionParameters() {
60+
return "self, request";
61+
}
62+
63+
@Override
64+
public String getResponseParameters() {
65+
return "self.net_client.get_response(request), request";
66+
}
5567
}

‎ds3-autogen-python/src/main/java/com/spectralogic/ds3autogen/python/generators/client/ClientModelGeneratorUtils.java‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
package com.spectralogic.ds3autogen.python.generators.client;
2+
3+
public interface ClientModelGeneratorUtils {
4+
5+
/** Gets a string containing a comma separated list of the command's function parameters */
6+
String getFunctionParameters();
7+
8+
/** Gets a string containing a comma separated list of parameters passed to the response constructor */
9+
String getResponseParameters();
10+
}

‎ds3-autogen-python/src/main/java/com/spectralogic/ds3autogen/python/generators/client/GetObjectCommandGenerator.java‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
/*
2+
* ******************************************************************************
3+
* Copyright 2014-2018 Spectra Logic Corporation. All Rights Reserved.
4+
* Licensed under the Apache License, Version 2.0 (the "License"). You may not use
5+
* this file except in compliance with the License. A copy of the License is located at
6+
*
7+
* http://www.apache.org/licenses/LICENSE-2.0
8+
*
9+
* or in the "license" file accompanying this file.
10+
* This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
11+
* CONDITIONS OF ANY KIND, either express or implied. See the License for the
12+
* specific language governing permissions and limitations under the License.
13+
* ****************************************************************************
14+
*/
15+
16+
package com.spectralogic.ds3autogen.python.generators.client;
17+
18+
public class GetObjectCommandGenerator extends BaseClientGenerator {
19+
20+
@Override
21+
public String getFunctionParameters() {
22+
return "self, request, buffer_size=1048576";
23+
}
24+
25+
@Override
26+
public String getResponseParameters() {
27+
return "self.net_client.get_response(request), request, buffer_size";
28+
}
29+
}

‎ds3-autogen-python/src/main/java/com/spectralogic/ds3autogen/python/generators/response/GetObjectResponseGenerator.java‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -31,12 +31,19 @@ public class GetObjectResponseGenerator extends BaseResponseGenerator {
3131
public String toParseResponsePayload(final Ds3Request ds3Request) {
3232
return "stream = self.request.stream\n" +
3333
pythonIndent(2) + "try:\n" +
34-
pythonIndent(3) + "bytes_read = response.read()\n" +
34+
pythonIndent(3) + "bytes_read = response.read(self.buffer_size)\n" +
3535
pythonIndent(3) + "while bytes_read:\n" +
3636
pythonIndent(4) + "stream.write(bytes_read)\n" +
37-
pythonIndent(4) + "bytes_read = response.read()\n" +
37+
pythonIndent(4) + "bytes_read = response.read(self.buffer_size)\n" +
3838
pythonIndent(2) + "finally:\n" +
3939
pythonIndent(3) + "stream.close()\n" +
4040
pythonIndent(3) + "response.close()\n";
4141
}
42+
43+
@Override
44+
public String toInitResponse() {
45+
return "def __init__(self, response, request, buffer_size=None):\n" +
46+
pythonIndent(2) + "self.buffer_size = buffer_size\n" +
47+
pythonIndent(2) + "super(self.__class__, self).__init__(response, request)\n";
48+
}
4249
}

‎ds3-autogen-python/src/main/java/com/spectralogic/ds3autogen/python/model/client/BaseClient.java‎

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,12 +21,22 @@ public class BaseClient {
2121
private final String requestType;
2222
private final String responseName;
2323
private final String documentation;
24+
private final String functionParams; /** comma separated list of function parameters */
25+
private final String responseParams; /** comma separated list of parameters passed to response */
2426

25-
public BaseClient(final String commandName, final String requestType, final String responseName, final String documentation) {
27+
public BaseClient(
28+
final String commandName,
29+
final String requestType,
30+
final String responseName,
31+
final String documentation,
32+
final String functionParams,
33+
final String responseParams) {
2634
this.commandName = commandName;
2735
this.requestType = requestType;
2836
this.responseName = responseName;
2937
this.documentation = documentation;
38+
this.functionParams = functionParams;
39+
this.responseParams = responseParams;
3040
}
3141

3242
public String getCommandName() {
@@ -44,4 +54,12 @@ public String getResponseName() {
4454
public String getDocumentation() {
4555
return documentation;
4656
}
57+
58+
public String getFunctionParams() {
59+
return functionParams;
60+
}
61+
62+
public String getResponseParams() {
63+
return responseParams;
64+
}
4765
}

‎ds3-autogen-python/src/main/kotlin/com/spectralogic/ds3autogen/python/PythonCodeGeneratorInterface.kt‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ import com.spectralogic.ds3autogen.api.models.apispec.Ds3Request
2121
import com.spectralogic.ds3autogen.api.models.apispec.Ds3Type
2222
import com.spectralogic.ds3autogen.api.models.docspec.Ds3DocSpec
2323
import com.spectralogic.ds3autogen.python.generators.request.BaseRequestGenerator
24+
import com.spectralogic.ds3autogen.python.generators.response.BaseResponseGenerator
2425
import com.spectralogic.ds3autogen.python.model.request.BaseRequest
2526
import freemarker.template.Configuration
2627
import freemarker.template.Template
@@ -48,4 +49,6 @@ interface PythonCodeGeneratorInterface {
4849

4950
fun toRequestModelList(ds3Requests: ImmutableList<Ds3Request>,
5051
docSpec: Ds3DocSpec) : ImmutableList<BaseRequest>
52+
53+
fun getGetObjectResponseGenerator() : BaseResponseGenerator
5154
}

‎ds3-autogen-python/src/main/resources/tmpls/python/commands/client/client_class.ftl‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,8 +9,8 @@ class Client(object):
99

1010
<#list clientCommands as cmd>
1111
${cmd.documentation}
12-
def ${cmd.commandName}(self, request):
12+
def ${cmd.commandName}(${cmd.functionParams}):
1313
if not isinstance(request, ${cmd.requestType}):
1414
raise TypeError('request for ${cmd.commandName} should be of type ${cmd.requestType} but was ' + request.__class__.__name__)
15-
return ${cmd.responseName}(self.net_client.get_response(request), request)
15+
return ${cmd.responseName}(${cmd.responseParams})
1616
</#list>

‎ds3-autogen-python/src/test/java/com.spectralogic.ds3autogen.python/PythonCodeGenerator_Test.java‎

Lines changed: 17 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,8 @@
2323
import com.spectralogic.ds3autogen.api.models.apispec.Ds3Type;
2424
import com.spectralogic.ds3autogen.api.models.enums.HttpVerb;
2525
import com.spectralogic.ds3autogen.docspec.Ds3DocSpecEmptyImpl;
26+
import com.spectralogic.ds3autogen.python.generators.client.BaseClientGenerator;
27+
import com.spectralogic.ds3autogen.python.generators.client.GetObjectCommandGenerator;
2628
import com.spectralogic.ds3autogen.python.generators.request.*;
2729
import com.spectralogic.ds3autogen.python.generators.response.BaseResponseGenerator;
2830
import com.spectralogic.ds3autogen.python.generators.response.GetObjectResponseGenerator;
@@ -172,22 +174,22 @@ public void toTypeDescriptorList_FullMap_Test() {
172174

173175
@Test
174176
public void getResponseGenerator_Test() {
175-
assertThat(getResponseGenerator(createDs3RequestTestData("com.test.EmptyRequest", Classification.spectrads3)),
177+
assertThat(generator.getResponseGenerator(createDs3RequestTestData("com.test.EmptyRequest", Classification.spectrads3)),
176178
instanceOf(BaseResponseGenerator.class));
177179

178-
assertThat(getResponseGenerator(getRequestAmazonS3GetObject()), instanceOf(GetObjectResponseGenerator.class));
180+
assertThat(generator.getResponseGenerator(getRequestAmazonS3GetObject()), instanceOf(GetObjectResponseGenerator.class));
179181

180-
assertThat(getResponseGenerator(getHeadObjectRequest()), instanceOf(HeadResponseGenerator.class));
182+
assertThat(generator.getResponseGenerator(getHeadObjectRequest()), instanceOf(HeadResponseGenerator.class));
181183

182-
assertThat(getResponseGenerator(getObjectsDetailsRequest()), instanceOf(PaginationResponseGenerator.class));
184+
assertThat(generator.getResponseGenerator(getObjectsDetailsRequest()), instanceOf(PaginationResponseGenerator.class));
183185

184-
assertThat(getResponseGenerator(getUsersSpectraS3Request()), instanceOf(PaginationResponseGenerator.class));
186+
assertThat(generator.getResponseGenerator(getUsersSpectraS3Request()), instanceOf(PaginationResponseGenerator.class));
185187
}
186188

187189
@Test
188190
public void toResponseModel_Test() {
189191
final Ds3Request request = getBucketRequest();
190-
final BaseResponse result = toResponseModel(request);
192+
final BaseResponse result = generator.toResponseModel(request);
191193
assertThat(result.getName(), is("GetBucketResponseHandler"));
192194
assertThat(result.getCodes().size(), is(1));
193195

@@ -198,20 +200,20 @@ public void toResponseModel_Test() {
198200

199201
@Test
200202
public void toResponseModelList_NullList_Test() {
201-
final ImmutableList<BaseResponse> result = toResponseModelList(null);
203+
final ImmutableList<BaseResponse> result = generator.toResponseModelList(null);
202204
assertThat(result.size(), is(0));
203205
}
204206

205207
@Test
206208
public void toResponseModelList_EmptyList_Test() {
207-
final ImmutableList<BaseResponse> result = toResponseModelList(ImmutableList.of());
209+
final ImmutableList<BaseResponse> result = generator.toResponseModelList(ImmutableList.of());
208210
assertThat(result.size(), is(0));
209211
}
210212

211213
@Test
212214
public void toResponseModelList_Test() {
213215
final ImmutableList<Ds3Request> requests = ImmutableList.of(getBucketRequest());
214-
final ImmutableList<BaseResponse> result = toResponseModelList(requests);
216+
final ImmutableList<BaseResponse> result = generator.toResponseModelList(requests);
215217
assertThat(result.size(), is(1));
216218
assertThat(result.get(0).getName(), is("GetBucketResponseHandler"));
217219
}
@@ -250,4 +252,10 @@ public void toClientCommands_Test() {
250252
assertThat(result.get(1).getResponseName(), is("PutBucketSpectraS3Response"));
251253
assertThat(result.get(1).getCommandName(), is("put_bucket_spectra_s3"));
252254
}
255+
256+
@Test
257+
public void getClientGenerator_Test() {
258+
assertThat(getClientGenerator(getRequestAmazonS3GetObject()), instanceOf(GetObjectCommandGenerator.class));
259+
assertThat(getClientGenerator(getRequestSpectraS3GetObject()), instanceOf(BaseClientGenerator.class));
260+
}
253261
}

‎ds3-autogen-python/src/test/java/com.spectralogic.ds3autogen.python/generators/client/BaseClientGenerator_Test.java‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,8 @@
2727

2828
public class BaseClientGenerator_Test {
2929

30+
private static final BaseClientGenerator generator = new BaseClientGenerator();
31+
3032
private static Ds3DocSpec getTestDocSpec() {
3133
return new Ds3DocSpecImpl(
3234
ImmutableMap.of(
@@ -57,4 +59,16 @@ public void toDocumentation_Test() {
5759
assertThat(toDocumentation("com.test.TestOneRequest", docSpec), is(expected));
5860
assertThat(toDocumentation("TestOneRequest", docSpec), is(expected));
5961
}
62+
63+
@Test
64+
public void getFunctionParametersTest() {
65+
final String expected = "self, request";
66+
assertThat(generator.getFunctionParameters(), is(expected));
67+
}
68+
69+
@Test
70+
public void getResponseParametersTest() {
71+
final String expected = "self.net_client.get_response(request), request";
72+
assertThat(generator.getResponseParameters(), is(expected));
73+
}
6074
}

0 commit comments

Comments
 (0)