Skip to content

Commit c7d4bc4

Browse files
committed
chore(demo/hbase-hdfs-load-cycling-data): Tidy up job scripts
1 parent 4745e4f commit c7d4bc4

File tree

2 files changed

+22
-5
lines changed

2 files changed

+22
-5
lines changed

demos/hbase-hdfs-load-cycling-data/create-hfile-and-import-to-hbase.yaml

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -28,16 +28,24 @@ spec:
2828
- mountPath: /stackable/conf/hbase-env.sh
2929
name: config-volume-hbase
3030
subPath: hbase-env.sh
31-
command: [ "bash", "-c", "/stackable/hbase/bin/hbase \
31+
command:
32+
- bash
33+
- -euo
34+
- pipefail
35+
- -c
36+
- |
37+
# https://hbase.apache.org/book.html#tools
38+
/stackable/hbase/bin/hbase \
3239
org.apache.hadoop.hbase.mapreduce.ImportTsv \
3340
-Dimporttsv.separator=, \
3441
-Dimporttsv.columns=HBASE_ROW_KEY,rideable_type,started_at,ended_at,start_station_name,start_station_id,end_station_name,end_station_id,start_lat,start_lng,end_lat,end_lng,member_casual \
3542
-Dimporttsv.bulk.output=hdfs://hdfs/data/hfile \
36-
cycling-tripdata hdfs://hdfs/data/raw/demo-cycling-tripdata.csv.gz \
37-
&& /stackable/hbase/bin/hbase \
43+
cycling-tripdata hdfs://hdfs/data/raw/demo-cycling-tripdata.csv.gz
44+
45+
/stackable/hbase/bin/hbase \
3846
org.apache.hadoop.hbase.tool.LoadIncrementalHFiles \
3947
hdfs://hdfs/data/hfile \
40-
cycling-tripdata" ] # https://hbase.apache.org/book.html#tools
48+
cycling-tripdata
4149
volumes:
4250
- name: config-volume-hbase
4351
configMap:

demos/hbase-hdfs-load-cycling-data/distcp-cycling-data.yaml

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,16 @@ spec:
2020
- name: HADOOP_CLASSPATH
2121
value: "/stackable/hadoop/share/hadoop/tools/lib/*.jar"
2222
# yamllint disable-line rule:line-length
23-
command: ["bash", "-c", "bin/hdfs dfs -mkdir -p /data/raw && bin/hadoop distcp -D fs.s3a.aws.credentials.provider=org.apache.hadoop.fs.s3a.AnonymousAWSCredentialsProvider s3a://public-backup-nyc-tlc/cycling-tripdata/demo-cycling-tripdata.csv.gz hdfs://hdfs/data/raw"]
23+
command:
24+
- bash
25+
- -euo
26+
- pipefail
27+
- -c
28+
- |
29+
bin/hdfs dfs -mkdir -p /data/raw
30+
bin/hadoop distcp -D fs.s3a.aws.credentials.provider=org.apache.hadoop.fs.s3a.AnonymousAWSCredentialsProvider \
31+
s3a://public-backup-nyc-tlc/cycling-tripdata/demo-cycling-tripdata.csv.gz \
32+
hdfs://hdfs/data/raw
2433
volumeMounts:
2534
- name: config-volume-hdfs
2635
mountPath: /stackable/conf/hdfs

0 commit comments

Comments
 (0)