Skip to content

Commit 5899a99

Browse files
authored
Add ETL/rETL demo (#141)
1 parent c19c235 commit 5899a99

File tree

2 files changed

+104
-1
lines changed

2 files changed

+104
-1
lines changed

deploy/samples/etl-retl.yaml

Lines changed: 104 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,104 @@
1+
2+
apiVersion: hoptimator.linkedin.com/v1alpha1
3+
kind: TableTemplate
4+
metadata:
5+
name: kafka-source-trigger-template
6+
spec:
7+
databases:
8+
- kafka-database
9+
methods:
10+
- Scan
11+
yaml: |
12+
apiVersion: hoptimator.linkedin.com/v1alpha1
13+
kind: TableTrigger
14+
metadata:
15+
name: {{name}}-trigger
16+
spec:
17+
schema: KAFKA
18+
table: {{table}}
19+
yaml: |
20+
apiVersion: batch/v1
21+
kind: Job
22+
metadata:
23+
name: {{name}}-job
24+
spec:
25+
template:
26+
spec:
27+
containers:
28+
- name: hello
29+
image: alpine/k8s:1.33.0
30+
command: ["bash", "-c", "echo {{name}}-trigger fired at `date`"]
31+
restartPolicy: Never
32+
backoffLimit: 4
33+
34+
---
35+
36+
## Simulate an upstream rETL job
37+
apiVersion: hoptimator.linkedin.com/v1alpha1
38+
kind: TableTemplate
39+
metadata:
40+
name: kafka-source-cron-template
41+
spec:
42+
databases:
43+
- kafka-database
44+
methods:
45+
- Scan
46+
yaml: |
47+
apiVersion: batch/v1
48+
kind: CronJob
49+
metadata:
50+
name: {{name}}-retl-cronjob
51+
spec:
52+
schedule: "* * * * *"
53+
jobTemplate:
54+
spec:
55+
ttlSecondsAfterFinished: 100
56+
template:
57+
spec:
58+
serviceAccountName: hoptimator-operator
59+
containers:
60+
- name: trigger
61+
image: alpine/k8s:1.33.0
62+
imagePullPolicy: IfNotPresent
63+
command:
64+
- /bin/bash
65+
- -c
66+
- >-
67+
echo "Pushing data to {{name}}...";
68+
kubectl patch tabletrigger {{name}}-trigger -p
69+
"{\"status\":{\"timestamp\": \"`date +%FT%TZ`\"}}"
70+
--subresource=status --type=merge
71+
restartPolicy: OnFailure
72+
73+
---
74+
75+
## Simulate a downstream ETL job
76+
apiVersion: hoptimator.linkedin.com/v1alpha1
77+
kind: JobTemplate
78+
metadata:
79+
name: kafka-etl-template
80+
spec:
81+
databases:
82+
- kafka-database
83+
yaml: |
84+
apiVersion: batch/v1
85+
kind: CronJob
86+
metadata:
87+
name: {{name}}-etl-job
88+
spec:
89+
schedule: "* * * * *"
90+
jobTemplate:
91+
spec:
92+
template:
93+
spec:
94+
containers:
95+
- name: hello
96+
image: alpine/k8s:1.33.0
97+
command:
98+
- bash
99+
- -c
100+
- >-
101+
echo "streaming Kafka topic {{table}} to disk...";
102+
restartPolicy: Never
103+
ttlSecondsAfterFinished: 100
104+

deploy/samples/tabletriggers.yaml

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,4 +21,3 @@ spec:
2121
backoffLimit: 4
2222
ttlSecondsAfterFinished: 90
2323
24-

0 commit comments

Comments
 (0)