Skip to content

Commit 9afda04

Browse files
123-komal123-komal
authored
Merge branch 'release/1.21' into snapshot-removal-6111
2 parents e8c90f2 + e018c46 commit 9afda04

File tree

4 files changed

+165
-28
lines changed

4 files changed

+165
-28
lines changed

‎docs/S3-batchsource.md‎

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,14 +19,27 @@ the credentials does not need to be provided.
1919
**Path:** Path to read from. For example, s3a://<bucket>/path/to/input
2020

2121
**Format:** Format of the data to read.
22-
The format must be one of 'avro', 'blob', 'csv', 'delimited', 'json', 'parquet', 'text', or 'tsv'.
22+
The format must be one of 'avro', 'blob', 'csv', 'delimited', 'json', 'parquet', 'text', 'tsv' or 'xls'.
2323
If the format is 'blob', every input file will be read into a separate record.
2424
The 'blob' format also requires a schema that contains a field named 'body' of type 'bytes'.
2525
If the format is 'text', the schema must contain a field named 'body' of type 'string'.
2626

27+
**Sample Size:** The maximum number of rows that will get investigated for automatic data type detection.
28+
The default value is 1000. This is only used when the format is 'xls'.
29+
30+
**Override:** A list of columns with the corresponding data types for whom the automatic data type detection gets
31+
skipped. This is only used when the format is 'xls'.
32+
33+
**Terminate Reading After Empty Row:** Specify whether to stop reading after encountering the first empty row. Defaults to false. When false the reader will read all rows in the sheet. This is only used when the format is 'xls'.
34+
35+
**Select Sheet Using:** Select the sheet by name or number. Default is 'Sheet Number'. This is only used when the format is 'xls'.
36+
37+
**Sheet Value:** The name/number of the sheet to read from. If not specified, the first sheet will be read.
38+
Sheet Numbers are 0 based, ie first sheet is 0. This is only used when the format is 'xls'.
39+
2740
**Delimiter:** Delimiter to use when the format is 'delimited'. This will be ignored for other formats.
2841

29-
**Use First Row as Header:** Whether to use first row as header. Supported formats are 'text', 'csv', 'tsv', 'delimited'.
42+
**Use First Row as Header:** Whether to use first row as header. Supported formats are 'text', 'csv', 'tsv', 'delimited', 'xls'.
3043

3144
**Authentication Method:** Authentication method to access S3. The default value is Access Credentials.
3245
IAM can only be used if the plugin is run in an AWS environment, such as on EMR.

‎pom.xml‎

Lines changed: 15 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -39,31 +39,11 @@
3939
<url>https://issues.cask.co/projects/CDAP</url>
4040
</issueManagement>
4141

42-
<distributionManagement>
43-
<repository>
44-
<id>sonatype.release</id>
45-
<url>https://oss.sonatype.org/service/local/staging/deploy/maven2</url>
46-
</repository>
47-
<snapshotRepository>
48-
<id>sonatype.snapshots</id>
49-
<url>https://oss.sonatype.org/content/repositories/snapshots</url>
50-
</snapshotRepository>
51-
</distributionManagement>
5242

5343
<repositories>
54-
<repository>
55-
<id>sonatype</id>
56-
<url>https://oss.sonatype.org/content/groups/public</url>
57-
<releases>
58-
<enabled>true</enabled>
59-
</releases>
60-
<snapshots>
61-
<enabled>false</enabled>
62-
</snapshots>
63-
</repository>
6444
<repository>
6545
<id>sonatype-snapshots</id>
66-
<url>https://oss.sonatype.org/content/repositories/snapshots</url>
46+
<url>https://central.sonatype.com/repository/maven-snapshots</url>
6747
<releases>
6848
<enabled>false</enabled>
6949
</releases>
@@ -75,8 +55,8 @@
7555

7656
<properties>
7757
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
78-
<cdap.version>6.11.0-SNAPSHOT</cdap.version>
79-
<cdap.plugin.version>2.13.0-SNAPSHOT</cdap.plugin.version>
58+
<cdap.version>6.11.0</cdap.version>
59+
<cdap.plugin.version>2.13.0</cdap.plugin.version>
8060
<data.pipeline.parent>system:cdap-data-pipeline[6.11.0-SNAPSHOT, 7.0.0-SNAPSHOT)</data.pipeline.parent>
8161
<data.stream.parent>system:cdap-data-streams[6.11.0-SNAPSHOT, 7.0.0-SNAPSHOT)</data.stream.parent>
8262
<hadoop.version>3.3.6</hadoop.version>
@@ -390,6 +370,17 @@
390370
<groupId>org.apache.maven.plugins</groupId>
391371
<artifactId>maven-antrun-plugin</artifactId>
392372
</plugin>
373+
<plugin>
374+
<groupId>org.sonatype.central</groupId>
375+
<artifactId>central-publishing-maven-plugin</artifactId>
376+
<version>0.8.0</version>
377+
<extensions>true</extensions>
378+
<configuration>
379+
<publishingServerId>sonatype.release</publishingServerId>
380+
<autoPublish>false</autoPublish>
381+
<ignorePublishedComponents>true</ignorePublishedComponents>
382+
</configuration>
383+
</plugin>
393384
</plugins>
394385
</build>
395386
<profiles>
@@ -541,7 +532,7 @@
541532
<dependency>
542533
<groupId>io.cdap.tests.e2e</groupId>
543534
<artifactId>cdap-e2e-framework</artifactId>
544-
<version>0.4.0-SNAPSHOT</version>
535+
<version>0.4.0</version>
545536
<scope>test</scope>
546537
</dependency>
547538
<dependency>

‎src/main/java/io/cdap/plugin/aws/s3/source/S3BatchSource.java‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -147,6 +147,9 @@ public static class S3BatchConfig extends AbstractFileSourceConfig {
147147
public static final String NAME_PATH = "path";
148148
private static final String NAME_FILE_SYSTEM_PROPERTIES = "fileSystemProperties";
149149
private static final String NAME_DELIMITER = "delimiter";
150+
private static final String NAME_SHEET = "sheet";
151+
private static final String NAME_SHEET_VALUE = "sheetValue";
152+
private static final String NAME_TERMINATE_IF_EMPTY_ROW = "terminateIfEmptyRow";
150153

151154
private static final Gson GSON = new Gson();
152155
private static final Type MAP_STRING_STRING_TYPE = new TypeToken<Map<String, String>>() { }.getType();
@@ -180,6 +183,36 @@ public static class S3BatchConfig extends AbstractFileSourceConfig {
180183
"fail, if credentials are incorrect. The default value is false.")
181184
private Boolean verifyCredentials;
182185

186+
@Macro
187+
@Nullable
188+
@Description("The maximum number of rows that will get investigated for automatic data type detection.")
189+
private Long sampleSize;
190+
191+
@Macro
192+
@Nullable
193+
@Description("A list of columns with the corresponding data types for whom the automatic data type detection gets" +
194+
" skipped.")
195+
private String override;
196+
197+
@Name(NAME_SHEET)
198+
@Macro
199+
@Nullable
200+
@Description("Select the sheet by name or number. Default is 'Sheet Number'.")
201+
private String sheet;
202+
203+
@Name(NAME_SHEET_VALUE)
204+
@Macro
205+
@Nullable
206+
@Description("The name/number of the sheet to read from. If not specified, the first sheet will be read." +
207+
"Sheet Numbers are 0 based, ie first sheet is 0.")
208+
private String sheetValue;
209+
210+
@Name(NAME_TERMINATE_IF_EMPTY_ROW)
211+
@Macro
212+
@Nullable
213+
@Description("Specify whether to stop reading after encountering the first empty row. Defaults to false.")
214+
private String terminateIfEmptyRow;
215+
183216
private S3BatchConfig(String path, @Nullable S3ConnectorConfig connection, String fileSystemProperties,
184217
Boolean verifyCredentials) {
185218
super();

‎widgets/S3-batchsource.json‎

Lines changed: 102 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -120,7 +120,8 @@
120120
"json",
121121
"parquet",
122122
"text",
123-
"tsv"
123+
"tsv",
124+
"xls"
124125
],
125126
"default": "text"
126127
},
@@ -135,6 +136,36 @@
135136
"plugin-method": "getSchema"
136137
}
137138
},
139+
{
140+
"widget-type": "number",
141+
"label": "Sample Size",
142+
"name": "sampleSize",
143+
"widget-attributes": {
144+
"default": "1000",
145+
"minimum": "1"
146+
}
147+
},
148+
{
149+
"widget-type": "keyvalue-dropdown",
150+
"label": "Override",
151+
"name": "override",
152+
"widget-attributes": {
153+
"key-placeholder": "Field Name",
154+
"value-placeholder": "Data Type",
155+
"dropdownOptions": [
156+
"boolean",
157+
"bytes",
158+
"double",
159+
"float",
160+
"int",
161+
"long",
162+
"string",
163+
"date",
164+
"time",
165+
"timestamp"
166+
]
167+
}
168+
},
138169
{
139170
"widget-type": "textbox",
140171
"label": "Delimiter",
@@ -174,6 +205,42 @@
174205
"label": "False"
175206
}
176207
}
208+
},
209+
{
210+
"widget-type": "toggle",
211+
"label": "Terminate Reading After Empty Row",
212+
"name": "terminateIfEmptyRow",
213+
"widget-attributes": {
214+
"default": "false",
215+
"on": {
216+
"value": "true",
217+
"label": "True"
218+
},
219+
"off": {
220+
"value": "false",
221+
"label": "False"
222+
}
223+
}
224+
},
225+
{
226+
"widget-type": "select",
227+
"label": "Select Sheet Using",
228+
"name": "sheet",
229+
"widget-attributes": {
230+
"values": [
231+
"Sheet Name",
232+
"Sheet Number"
233+
],
234+
"default": "Sheet Number"
235+
}
236+
},
237+
{
238+
"widget-type": "textbox",
239+
"label": "Sheet Value",
240+
"name": "sheetValue",
241+
"widget-attributes": {
242+
"default": "0"
243+
}
177244
}
178245
]
179246
},
@@ -674,13 +741,46 @@
674741
{
675742
"name": "skipHeader",
676743
"condition": {
677-
"expression": "format == 'delimited' || format == 'csv' || format == 'tsv'"
744+
"expression": "format == 'delimited' || format == 'csv' || format == 'tsv' || format == 'xls'"
678745
},
679746
"show": [
680747
{
681748
"name": "skipHeader"
682749
}
683750
]
751+
},
752+
{
753+
"name": "sheet",
754+
"condition": {
755+
"expression": "format == 'xls'"
756+
},
757+
"show": [
758+
{
759+
"name": "sheet"
760+
}
761+
]
762+
},
763+
{
764+
"name": "sheetValue",
765+
"condition": {
766+
"expression": "format == 'xls'"
767+
},
768+
"show": [
769+
{
770+
"name": "sheetValue"
771+
}
772+
]
773+
},
774+
{
775+
"name": "terminateIfEmptyRow",
776+
"condition": {
777+
"expression": "format == 'xls'"
778+
},
779+
"show": [
780+
{
781+
"name": "terminateIfEmptyRow"
782+
}
783+
]
684784
}
685785
]
686786
}

0 commit comments

Comments
 (0)