feat: s3 support example

Author: purbanow

build.gradle

@@ -23,6 +23,7 @@ repositories {
 dependencies {
     testCompile group: 'junit', name: 'junit', version: '4.12'
     compile files('beam-sdks-java-io-snowflake-2.22.0-SNAPSHOT.jar')
+    compile 'org.apache.beam:beam-sdks-java-io-amazon-web-services:2.22.0'
     compile group: 'org.apache.beam', name: 'beam-sdks-java-core', version: '2.22.0'
     compile group: 'org.apache.beam', name: 'beam-runners-direct-java', version: '2.22.0'
     compile group: 'org.apache.beam', name: 'beam-runners-google-cloud-dataflow-java', version: '2.22.0'

src/main/java/batching/SnowflakeWordCountOptions.java

@@ -4,11 +4,12 @@
 import org.apache.beam.sdk.options.Default;
 import org.apache.beam.sdk.options.Description;
 import org.apache.beam.sdk.options.Validation.Required;
+import org.apache.beam.sdk.io.aws.options.S3Options;
 
 /**
  * Supported PipelineOptions used in provided examples.
  */
-public interface SnowflakeWordCountOptions extends SnowflakePipelineOptions {
+public interface SnowflakeWordCountOptions extends SnowflakePipelineOptions, S3Options {
 
     @Description("Path of the file to read from")
     @Default.String("gs://apache-beam-samples/shakespeare/kinglear.txt")
@@ -21,4 +22,16 @@ public interface SnowflakeWordCountOptions extends SnowflakePipelineOptions {
     String getOutput();
 
     void setOutput(String value);
+
+    @Description("AWS Access Key")
+    @Required
+    String getAwsAccessKey();
+
+    void setAwsAccessKey(String awsAccessKey);
+
+    @Description("AWS secret key")
+    @Required
+    String getAwsSecretKey();
+
+    void setAwsSecretKey(String awsSecretKey);
 }

src/main/java/batching/WordCountExample.java

@@ -23,10 +23,13 @@
 import org.apache.beam.sdk.values.PBegin;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.PDone;
+import com.amazonaws.auth.AWSCredentials;
+import com.amazonaws.auth.AWSStaticCredentialsProvider;
+import com.amazonaws.auth.BasicAWSCredentials;
 
 /**
  * An example that contains batch writing and reading from Snowflake. Inspired by Apache Beam/WordCount-example(https://github.com/apache/beam/blob/master/examples/java/src/main/java/org/apache/beam/examples/WordCount.java)
- *
+ * <p>
  * Check main README for more information.
  */
 public class WordCountExample {
@@ -35,10 +38,18 @@ public static void main(String[] args) {
         SnowflakeWordCountOptions options =
                 PipelineOptionsFactory.fromArgs(args).withValidation().as(SnowflakeWordCountOptions.class);
 
+        options = parseAwsOptions(options);
+
         runWritingToSnowflake(options);
         runReadingFromSnowflake(options);
     }
 
+    private static SnowflakeWordCountOptions parseAwsOptions(SnowflakeWordCountOptions options) {
+        AWSCredentials awsCredentials = new BasicAWSCredentials(options.getAwsAccessKey(), options.getAwsSecretKey());
+        options.setAwsCredentialsProvider(new AWSStaticCredentialsProvider(awsCredentials));
+
+        return options;
+    }
 
     private static void runWritingToSnowflake(SnowflakeWordCountOptions options) {
         Pipeline p = Pipeline.create(options);

src/main/java/streaming/TaxiRidesExample.java

@@ -44,7 +44,6 @@ public static void main(String[] args) {
                                 .withSnowPipe(options.getSnowPipe())
                                 .withFileNameTemplate(UUID.randomUUID().toString())
                                 .withFlushTimeLimit(Duration.millis(3000))
-                                .withDebugMode(SnowflakeIO.StreamingLogLevel.INFO)
                                 .withFlushRowLimit(100)
                                 .withQuotationMark("")
                                 .withShardsNumber(1));