data-engineering-zoomcamp/07-streaming/workshop/flink-config.yaml at main · amany-mm/data-engineering-zoomcamp · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
# Custom Flink config for PyFlink workshop.
# Original: https://github.com/apache/flink/blob/release-2.2/flink-dist/src/main/resources/config.yaml
# Changes from default:
#   1. Added taskmanager.memory.jvm-metaspace.size: 512m (PyFlink needs more metaspace)
#   2. Removed --add-exports=jdk.compiler/... from env.java.opts.all
#      (jdk.compiler module is not present in the JRE, causing warnings on every command)

blob:
  server:
    port: '6124'
taskmanager:
  memory:
    process:
      size: 1728m
    jvm-metaspace:
      size: 512m  # added for PyFlink
  bind-host: 0.0.0.0
  numberOfTaskSlots: 15
jobmanager:
  execution:
    failover-strategy: region
  rpc:
    address: jobmanager
    port: 6123
  memory:
    process:
      size: 1600m
  bind-host: 0.0.0.0
query:
  server:
    port: '6125'
parallelism:
  default: 1
rest:
  address: 0.0.0.0
env:
  java:
    opts:
      all: >-
        --add-exports=java.rmi/sun.rmi.registry=ALL-UNNAMED
        --add-exports=java.security.jgss/sun.security.krb5=ALL-UNNAMED
        --add-opens=java.base/java.lang=ALL-UNNAMED
        --add-opens=java.base/java.net=ALL-UNNAMED
        --add-opens=java.base/java.io=ALL-UNNAMED
        --add-opens=java.base/java.nio=ALL-UNNAMED
        --add-opens=java.base/sun.nio.ch=ALL-UNNAMED
        --add-opens=java.base/java.lang.reflect=ALL-UNNAMED
        --add-opens=java.base/java.text=ALL-UNNAMED
        --add-opens=java.base/java.time=ALL-UNNAMED
        --add-opens=java.base/java.util=ALL-UNNAMED
        --add-opens=java.base/java.util.concurrent=ALL-UNNAMED
        --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED
        --add-opens=java.base/java.util.concurrent.locks=ALL-UNNAMED