diff --git a/.gitignore b/.gitignore index 6356b9f..3875d2a 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,7 @@ # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 # User-specific stuff +.idea/** .idea/**/workspace.xml .idea/**/tasks.xml .idea/**/usage.statistics.xml @@ -97,4 +98,5 @@ fabric.properties *.rar # virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml -hs_err_pid* \ No newline at end of file +hs_err_pid* +target/** diff --git a/beam_df_dedup/.idea/codeStyles/codeStyleConfig.xml b/beam_df_dedup/.idea/codeStyles/codeStyleConfig.xml deleted file mode 100644 index a55e7a1..0000000 --- a/beam_df_dedup/.idea/codeStyles/codeStyleConfig.xml +++ /dev/null @@ -1,5 +0,0 @@ - - - - \ No newline at end of file diff --git a/beam_df_dedup/.idea/compiler.xml b/beam_df_dedup/.idea/compiler.xml deleted file mode 100644 index 4a14a77..0000000 --- a/beam_df_dedup/.idea/compiler.xml +++ /dev/null @@ -1,16 +0,0 @@ - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/beam_df_dedup/.idea/misc.xml b/beam_df_dedup/.idea/misc.xml deleted file mode 100644 index 08c6a01..0000000 --- a/beam_df_dedup/.idea/misc.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - \ No newline at end of file diff --git a/beam_df_dedup/.idea/modules.xml b/beam_df_dedup/.idea/modules.xml deleted file mode 100644 index 4e92467..0000000 --- a/beam_df_dedup/.idea/modules.xml +++ /dev/null @@ -1,8 +0,0 @@ - - - - - - - - \ No newline at end of file diff --git a/beam_df_dedup/.idea/vcs.xml b/beam_df_dedup/.idea/vcs.xml deleted file mode 100644 index 2e3f692..0000000 --- a/beam_df_dedup/.idea/vcs.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/beam_df_dedup/src/main/java/com/google/examples/dfdedup/DedupWithDistinct.java b/beam_df_dedup/src/main/java/com/google/examples/dfdedup/DedupWithDistinct.java index a4d72cd..2b120e1 100644 --- a/beam_df_dedup/src/main/java/com/google/examples/dfdedup/DedupWithDistinct.java +++ b/beam_df_dedup/src/main/java/com/google/examples/dfdedup/DedupWithDistinct.java @@ -30,7 +30,7 @@ public class DedupWithDistinct { private static final Logger LOG = LoggerFactory.getLogger(DedupWithDistinct.class); private static PCollection dedupWithDistinct(PCollection stream) { - + // [START FIXED_WINDOW_DISTINCT] PCollection windowedStream = stream.apply("WindowForDistinct", Window.into(FixedWindows.of(Duration.standardSeconds(10))) @@ -43,12 +43,12 @@ private static PCollection dedupWithDistinct(PCollection strea windowedStream.apply(Distinct.withRepresentativeValueFn( (Message m) -> (m.getLogicalId())) .withRepresentativeType(TypeDescriptor.of(String.class))); - + // [END FIXED_WINDOW_DISTINCT] return dedupedStream; } private static PCollection dedupWithDistinctGlobal(PCollection stream) { - + // [START GLOBAL_WINDOW_DISTINCT] PCollection windowedStream = stream.apply("WindowForDistinct", Window.into(new GlobalWindows()) @@ -61,7 +61,7 @@ private static PCollection dedupWithDistinctGlobal(PCollection windowedStream.apply(Distinct.withRepresentativeValueFn( (Message m) -> (m.getLogicalId())) .withRepresentativeType(TypeDescriptor.of(String.class))); - + // [END GLOBAL_WINDOW_DISTINCT] return dedupedStream; } diff --git a/beam_df_dedup/target/maven-status/maven-compiler-plugin/compile/default-compile/createdFiles.lst b/beam_df_dedup/target/maven-status/maven-compiler-plugin/compile/default-compile/createdFiles.lst deleted file mode 100644 index 065acdd..0000000 --- a/beam_df_dedup/target/maven-status/maven-compiler-plugin/compile/default-compile/createdFiles.lst +++ /dev/null @@ -1,22 +0,0 @@ -com/google/examples/dfdedup/common/MessageToJsonConverter.class -com/google/examples/dfdedup/common/JsonToMessageConverter$1.class -com/google/examples/dfdedup/common/StateBasedDistinct$StateKeeperFn.class -com/google/examples/dfdedup/common/MessageToJsonConverter$1.class -com/google/examples/dfdedup/common/StateBasedDistinct.class -com/google/examples/dfdedup/common/AssignDfUniqueIds.class -com/google/examples/dfdedup/DedupWithState.class -com/google/examples/dfdedup/common/JsonToMessageConverter.class -com/google/examples/dfdedup/DedupWithStateAndGC.class -com/google/examples/dfdedup/DedupWithPubSubIO.class -com/google/examples/dfdedup/common/ReadFromPubSub.class -com/google/examples/dfdedup/common/StateBasedDistinctWithTimer$1.class -com/google/examples/dfdedup/common/AssignDfUniqueIds$1.class -com/google/examples/dfdedup/common/AssignDfUniqueIds$IdAssignmentDoFn.class -com/google/examples/dfdedup/common/StateBasedDistinctWithTimer$StateKeeperWithTimerFn.class -com/google/examples/dfdedup/common/WriteMessagesToBQ.class -com/google/examples/dfdedup/DedupWithDistinct.class -com/google/examples/dfdedup/common/DedupPipelineOptions.class -com/google/examples/dfdedup/common/Message.class -com/google/examples/dfdedup/common/StateBasedDistinct$1.class -com/google/examples/dfdedup/common/ReadFromPubSub$1.class -com/google/examples/dfdedup/common/StateBasedDistinctWithTimer.class diff --git a/beam_df_dedup/target/maven-status/maven-compiler-plugin/compile/default-compile/inputFiles.lst b/beam_df_dedup/target/maven-status/maven-compiler-plugin/compile/default-compile/inputFiles.lst deleted file mode 100644 index 9903304..0000000 --- a/beam_df_dedup/target/maven-status/maven-compiler-plugin/compile/default-compile/inputFiles.lst +++ /dev/null @@ -1,13 +0,0 @@ -/mnt/c/code/gcloud/solutions/df-dedup/beam_df_dedup/src/main/java/com/google/examples/dfdedup/common/MessageToJsonConverter.java -/mnt/c/code/gcloud/solutions/df-dedup/beam_df_dedup/src/main/java/com/google/examples/dfdedup/common/JsonToMessageConverter.java -/mnt/c/code/gcloud/solutions/df-dedup/beam_df_dedup/src/main/java/com/google/examples/dfdedup/common/StateBasedDistinct.java -/mnt/c/code/gcloud/solutions/df-dedup/beam_df_dedup/src/main/java/com/google/examples/dfdedup/common/StateBasedDistinctWithTimer.java -/mnt/c/code/gcloud/solutions/df-dedup/beam_df_dedup/src/main/java/com/google/examples/dfdedup/common/ReadFromPubSub.java -/mnt/c/code/gcloud/solutions/df-dedup/beam_df_dedup/src/main/java/com/google/examples/dfdedup/DedupWithState.java -/mnt/c/code/gcloud/solutions/df-dedup/beam_df_dedup/src/main/java/com/google/examples/dfdedup/common/Message.java -/mnt/c/code/gcloud/solutions/df-dedup/beam_df_dedup/src/main/java/com/google/examples/dfdedup/common/AssignDfUniqueIds.java -/mnt/c/code/gcloud/solutions/df-dedup/beam_df_dedup/src/main/java/com/google/examples/dfdedup/common/DedupPipelineOptions.java -/mnt/c/code/gcloud/solutions/df-dedup/beam_df_dedup/src/main/java/com/google/examples/dfdedup/common/WriteMessagesToBQ.java -/mnt/c/code/gcloud/solutions/df-dedup/beam_df_dedup/src/main/java/com/google/examples/dfdedup/DedupWithStateAndGC.java -/mnt/c/code/gcloud/solutions/df-dedup/beam_df_dedup/src/main/java/com/google/examples/dfdedup/DedupWithDistinct.java -/mnt/c/code/gcloud/solutions/df-dedup/beam_df_dedup/src/main/java/com/google/examples/dfdedup/DedupWithPubSubIO.java