Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839

# User-specific stuff
.idea/**
.idea/**/workspace.xml
.idea/**/tasks.xml
.idea/**/usage.statistics.xml
Expand Down Expand Up @@ -97,4 +98,5 @@ fabric.properties
*.rar

# virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml
hs_err_pid*
hs_err_pid*
target/**
5 changes: 0 additions & 5 deletions beam_df_dedup/.idea/codeStyles/codeStyleConfig.xml

This file was deleted.

16 changes: 0 additions & 16 deletions beam_df_dedup/.idea/compiler.xml

This file was deleted.

13 changes: 0 additions & 13 deletions beam_df_dedup/.idea/misc.xml

This file was deleted.

8 changes: 0 additions & 8 deletions beam_df_dedup/.idea/modules.xml

This file was deleted.

6 changes: 0 additions & 6 deletions beam_df_dedup/.idea/vcs.xml

This file was deleted.

Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ public class DedupWithDistinct {
private static final Logger LOG = LoggerFactory.getLogger(DedupWithDistinct.class);

private static PCollection<Message> dedupWithDistinct(PCollection<Message> stream) {

// [START FIXED_WINDOW_DISTINCT]
PCollection<Message> windowedStream =
stream.apply("WindowForDistinct",
Window.<Message>into(FixedWindows.of(Duration.standardSeconds(10)))
Expand All @@ -43,12 +43,12 @@ private static PCollection<Message> dedupWithDistinct(PCollection<Message> strea
windowedStream.apply(Distinct.withRepresentativeValueFn(
(Message m) -> (m.getLogicalId()))
.withRepresentativeType(TypeDescriptor.of(String.class)));

// [END FIXED_WINDOW_DISTINCT]
return dedupedStream;
}

private static PCollection<Message> dedupWithDistinctGlobal(PCollection<Message> stream) {

// [START GLOBAL_WINDOW_DISTINCT]
PCollection<Message> windowedStream =
stream.apply("WindowForDistinct",
Window.<Message>into(new GlobalWindows())
Expand All @@ -61,7 +61,7 @@ private static PCollection<Message> dedupWithDistinctGlobal(PCollection<Message>
windowedStream.apply(Distinct.withRepresentativeValueFn(
(Message m) -> (m.getLogicalId()))
.withRepresentativeType(TypeDescriptor.of(String.class)));

// [END GLOBAL_WINDOW_DISTINCT]
return dedupedStream;
}

Expand Down

This file was deleted.

This file was deleted.