Skip to content

Commit 964c654

Browse files
deryrahmanderyrahman
committed
Fix: source entrypoint with custom shell (#751)
* fix: adding exec_entrypoint script * Revert "fix: adding exec_entrypoint script" This reverts commit ce373c2. * feat: add cmds for custom shell * refactor: rename entrypoint cmds and script * remove: join template func
1 parent 83ece95 commit 964c654

File tree

12 files changed

+140
-61
lines changed

12 files changed

+140
-61
lines changed

‎ext/scheduler/airflow/dag/compiler_test.go‎

Lines changed: 28 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -157,36 +157,48 @@ func (m mockPluginRepo) GetByName(name string) (*plugin.Plugin, error) {
157157
func setupPluginRepo() mockPluginRepo {
158158
execUnit := new(mock.YamlMod)
159159
execUnit.On("PluginInfo").Return(&plugin.Info{
160-
Name: "bq-bq",
161-
Image: "example.io/namespace/bq2bq-executor:latest",
162-
Entrypoint: "python3 /opt/bumblebee/main.py",
160+
Name: "bq-bq",
161+
Image: "example.io/namespace/bq2bq-executor:latest",
162+
Entrypoint: plugin.Entrypoint{
163+
Shell: "/bin/bash",
164+
Script: "python3 /opt/bumblebee/main.py",
165+
},
163166
}, nil)
164167

165168
transporterHook := "transporter"
166169
hookUnit := new(mock.YamlMod)
167170
hookUnit.On("PluginInfo").Return(&plugin.Info{
168-
Name: transporterHook,
169-
HookType: plugin.HookTypePre,
170-
Image: "example.io/namespace/transporter-executor:latest",
171-
Entrypoint: "java -cp /opt/transporter/transporter.jar:/opt/transporter/jolokia-jvm-agent.jar -javaagent:jolokia-jvm-agent.jar=port=7777,host=0.0.0.0 com.gojek.transporter.Main",
172-
DependsOn: []string{"predator"},
171+
Name: transporterHook,
172+
HookType: plugin.HookTypePre,
173+
Image: "example.io/namespace/transporter-executor:latest",
174+
Entrypoint: plugin.Entrypoint{
175+
Shell: "/bin/sh",
176+
Script: "java -cp /opt/transporter/transporter.jar:/opt/transporter/jolokia-jvm-agent.jar -javaagent:jolokia-jvm-agent.jar=port=7777,host=0.0.0.0 com.gojek.transporter.Main",
177+
},
178+
DependsOn: []string{"predator"},
173179
}, nil)
174180

175181
predatorHook := "predator"
176182
hookUnit2 := new(mock.YamlMod)
177183
hookUnit2.On("PluginInfo").Return(&plugin.Info{
178-
Name: predatorHook,
179-
HookType: plugin.HookTypePost,
180-
Image: "example.io/namespace/predator-image:latest",
181-
Entrypoint: "predator ${SUB_COMMAND} -s ${PREDATOR_URL} -u \"${BQ_PROJECT}.${BQ_DATASET}.${BQ_TABLE}\"",
184+
Name: predatorHook,
185+
HookType: plugin.HookTypePost,
186+
Image: "example.io/namespace/predator-image:latest",
187+
Entrypoint: plugin.Entrypoint{
188+
Shell: "/bin/sh",
189+
Script: "predator ${SUB_COMMAND} -s ${PREDATOR_URL} -u \"${BQ_PROJECT}.${BQ_DATASET}.${BQ_TABLE}\"",
190+
},
182191
}, nil)
183192

184193
hookUnit3 := new(mock.YamlMod)
185194
hookUnit3.On("PluginInfo").Return(&plugin.Info{
186-
Name: "failureHook",
187-
HookType: plugin.HookTypeFail,
188-
Image: "example.io/namespace/failure-hook-image:latest",
189-
Entrypoint: "sleep 5",
195+
Name: "failureHook",
196+
HookType: plugin.HookTypeFail,
197+
Image: "example.io/namespace/failure-hook-image:latest",
198+
Entrypoint: plugin.Entrypoint{
199+
Shell: "/bin/sh",
200+
Script: "sleep 5",
201+
},
190202
}, nil)
191203

192204
repo := mockPluginRepo{plugins: []*plugin.Plugin{

‎ext/scheduler/airflow/dag/dag.py.tmpl‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -102,12 +102,12 @@ IMAGE_PULL_POLICY = "IfNotPresent"
102102
INIT_CONTAINER_IMAGE = "odpf/optimus:{{.Version}}"
103103
INIT_CONTAINER_ENTRYPOINT = "/opt/entrypoint_init_container.sh"
104104

105-
def get_entrypoint_cmd(plugin_entrypoint):
105+
def get_entrypoint_cmd(plugin_entrypoint_script):
106106
path_config = JOB_DIR + "/in/.env"
107107
path_secret = JOB_DIR + "/in/.secret"
108108
entrypoint = "set -o allexport; source {path_config}; set +o allexport; cat {path_config}; ".format(path_config=path_config)
109109
entrypoint += "set -o allexport; source {path_secret}; set +o allexport; ".format(path_secret=path_secret)
110-
return entrypoint + plugin_entrypoint
110+
return entrypoint + plugin_entrypoint_script
111111

112112
volume = k8s.V1Volume(
113113
name='asset-volume',
@@ -150,8 +150,8 @@ init_container = k8s.V1Container(
150150
image_pull_policy=IMAGE_PULL_POLICY,
151151
namespace=conf.get('kubernetes', 'namespace', fallback="default"),
152152
image={{ .Task.Image | quote}},
153-
cmds=["/bin/sh"],
154-
arguments=["-c", get_entrypoint_cmd("""{{.Task.Entrypoint}} """)],
153+
cmds=["{{.Task.Entrypoint.Shell}}", "-c"],
154+
arguments=[get_entrypoint_cmd("""{{.Task.Entrypoint.Script}} """)],
155155
name="{{ .Task.Name | replace "_" "-" }}",
156156
task_id={{ .Task.Name | quote}},
157157
get_logs=True,
@@ -193,8 +193,8 @@ hook_{{$hookName}} = SuperKubernetesPodOperator(
193193
image_pull_policy=IMAGE_PULL_POLICY,
194194
namespace=conf.get('kubernetes', 'namespace', fallback="default"),
195195
image="{{ $t.Image }}",
196-
cmds=["/bin/sh"],
197-
arguments=["-c", get_entrypoint_cmd("""{{ $t.Entrypoint }} """)],
196+
cmds=["{{$t.Entrypoint.Shell}}", "-c"],
197+
arguments=[get_entrypoint_cmd("""{{$t.Entrypoint.Script}} """)],
198198
name="hook_{{ $t.Name | replace "_" "-" }}",
199199
task_id="hook_{{ $t.Name }}",
200200
get_logs=True,

‎ext/scheduler/airflow/dag/expected_dag.py‎

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -72,12 +72,12 @@
7272
INIT_CONTAINER_IMAGE = "odpf/optimus:dev"
7373
INIT_CONTAINER_ENTRYPOINT = "/opt/entrypoint_init_container.sh"
7474

75-
def get_entrypoint_cmd(plugin_entrypoint):
75+
def get_entrypoint_cmd(plugin_entrypoint_script):
7676
path_config = JOB_DIR + "/in/.env"
7777
path_secret = JOB_DIR + "/in/.secret"
7878
entrypoint = "set -o allexport; source {path_config}; set +o allexport; cat {path_config}; ".format(path_config=path_config)
7979
entrypoint += "set -o allexport; source {path_secret}; set +o allexport; ".format(path_secret=path_secret)
80-
return entrypoint + plugin_entrypoint
80+
return entrypoint + plugin_entrypoint_script
8181

8282
volume = k8s.V1Volume(
8383
name='asset-volume',
@@ -119,8 +119,8 @@ def get_entrypoint_cmd(plugin_entrypoint):
119119
image_pull_policy=IMAGE_PULL_POLICY,
120120
namespace=conf.get('kubernetes', 'namespace', fallback="default"),
121121
image="example.io/namespace/bq2bq-executor:latest",
122-
cmds=["/bin/sh"],
123-
arguments=["-c", get_entrypoint_cmd("""python3 /opt/bumblebee/main.py """)],
122+
cmds=["/bin/bash", "-c"],
123+
arguments=[get_entrypoint_cmd("""python3 /opt/bumblebee/main.py """)],
124124
name="bq-bq",
125125
task_id="bq-bq",
126126
get_logs=True,
@@ -156,8 +156,8 @@ def get_entrypoint_cmd(plugin_entrypoint):
156156
image_pull_policy=IMAGE_PULL_POLICY,
157157
namespace=conf.get('kubernetes', 'namespace', fallback="default"),
158158
image="example.io/namespace/transporter-executor:latest",
159-
cmds=["/bin/sh"],
160-
arguments=["-c", get_entrypoint_cmd("""java -cp /opt/transporter/transporter.jar:/opt/transporter/jolokia-jvm-agent.jar -javaagent:jolokia-jvm-agent.jar=port=7777,host=0.0.0.0 com.gojek.transporter.Main """)],
159+
cmds=["/bin/sh", "-c"],
160+
arguments=[get_entrypoint_cmd("""java -cp /opt/transporter/transporter.jar:/opt/transporter/jolokia-jvm-agent.jar -javaagent:jolokia-jvm-agent.jar=port=7777,host=0.0.0.0 com.gojek.transporter.Main """)],
161161
name="hook_transporter",
162162
task_id="hook_transporter",
163163
get_logs=True,
@@ -189,8 +189,8 @@ def get_entrypoint_cmd(plugin_entrypoint):
189189
image_pull_policy=IMAGE_PULL_POLICY,
190190
namespace=conf.get('kubernetes', 'namespace', fallback="default"),
191191
image="example.io/namespace/predator-image:latest",
192-
cmds=["/bin/sh"],
193-
arguments=["-c", get_entrypoint_cmd("""predator ${SUB_COMMAND} -s ${PREDATOR_URL} -u "${BQ_PROJECT}.${BQ_DATASET}.${BQ_TABLE}" """)],
192+
cmds=["/bin/sh", "-c"],
193+
arguments=[get_entrypoint_cmd("""predator ${SUB_COMMAND} -s ${PREDATOR_URL} -u "${BQ_PROJECT}.${BQ_DATASET}.${BQ_TABLE}" """)],
194194
name="hook_predator",
195195
task_id="hook_predator",
196196
get_logs=True,
@@ -222,8 +222,8 @@ def get_entrypoint_cmd(plugin_entrypoint):
222222
image_pull_policy=IMAGE_PULL_POLICY,
223223
namespace=conf.get('kubernetes', 'namespace', fallback="default"),
224224
image="example.io/namespace/failure-hook-image:latest",
225-
cmds=["/bin/sh"],
226-
arguments=["-c", get_entrypoint_cmd("""sleep 5 """)],
225+
cmds=["/bin/sh", "-c"],
226+
arguments=[get_entrypoint_cmd("""sleep 5 """)],
227227
name="hook_failureHook",
228228
task_id="hook_failureHook",
229229
get_logs=True,

‎ext/scheduler/airflow/dag/models.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ type TemplateContext struct {
3333
type Task struct {
3434
Name string
3535
Image string
36-
Entrypoint string
36+
Entrypoint plugin.Entrypoint
3737
}
3838

3939
func PrepareTask(job *scheduler.Job, pluginRepo PluginRepo) (Task, error) {
@@ -54,7 +54,7 @@ func PrepareTask(job *scheduler.Job, pluginRepo PluginRepo) (Task, error) {
5454
type Hook struct {
5555
Name string
5656
Image string
57-
Entrypoint string
57+
Entrypoint plugin.Entrypoint
5858
IsFailHook bool
5959
}
6060

‎plugin/yaml/plugin.go‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import (
55
"fmt"
66
"io"
77
"os"
8+
"strings"
89

910
"github.com/hashicorp/go-hclog"
1011
"github.com/spf13/afero"
@@ -104,6 +105,14 @@ func NewPluginSpec(pluginPath string) (*PluginSpec, error) {
104105
if err := yaml.UnmarshalStrict(pluginBytes, &plugin); err != nil {
105106
return &plugin, err
106107
}
108+
// default values
109+
if plugin.Info.Entrypoint.Shell == "" {
110+
plugin.Info.Entrypoint.Shell = "/bin/sh"
111+
}
112+
113+
// standardize script value
114+
script := plugin.Info.Entrypoint.Script
115+
plugin.Info.Entrypoint.Script = strings.ReplaceAll(script, "\n", "; ")
107116
return &plugin, nil
108117
}
109118

‎plugin/yaml/plugin_test.go‎

Lines changed: 19 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ import (
1616
type mockYamlMod struct {
1717
Name string
1818
Image string
19-
Entrypoint string
19+
Entrypoint plugin.Entrypoint
2020
PluginVersion string
2121
PluginType string
2222
}
@@ -51,10 +51,13 @@ func TestYamlPlugin(t *testing.T) {
5151
testYamlPluginPath := "tests/sample_plugin.yaml" // success
5252
testYamlPluginName := "bq2bqtest"
5353
expectedInfo := &plugin.Info{
54-
Name: "bq2bqtest",
55-
Description: "Testing",
56-
Image: "docker.io/odpf/optimus-task-bq2bq-executor:latest",
57-
Entrypoint: "sleep 100",
54+
Name: "bq2bqtest",
55+
Description: "Testing",
56+
Image: "docker.io/odpf/optimus-task-bq2bq-executor:latest",
57+
Entrypoint: plugin.Entrypoint{
58+
Shell: "/bin/bash",
59+
Script: "sleep 100; sleep 150",
60+
},
5861
PluginType: "task",
5962
PluginMods: []plugin.Mod{"cli"},
6063
PluginVersion: "latest",
@@ -168,12 +171,20 @@ func TestYamlPlugin(t *testing.T) {
168171
assert.NoError(t, err)
169172
assert.NotEmpty(t, repo.GetAll())
170173
})
174+
t.Run("should use default when entrypoint cmds is empty", func(t *testing.T) {
175+
pluginSpec, err := yaml.NewPluginSpec("tests/sample_plugin_without_shell.yaml")
176+
assert.NoError(t, err)
177+
assert.NotEmpty(t, pluginSpec)
178+
assert.Equal(t, "/bin/sh", pluginSpec.Entrypoint.Shell)
179+
})
171180
t.Run("should returns error when load yaml when same name exists", func(t *testing.T) {
172181
repoWithBinayPlugin := models.NewPluginRepository()
173182
err := repoWithBinayPlugin.AddYaml(&mockYamlMod{
174-
Name: testYamlPluginName,
175-
Image: "sdsd",
176-
Entrypoint: "sleep 100",
183+
Name: testYamlPluginName,
184+
Image: "sdsd",
185+
Entrypoint: plugin.Entrypoint{
186+
Script: "sleep 100",
187+
},
177188
PluginVersion: "asdasd",
178189
PluginType: plugin.TypeTask.String(),
179190
})

‎plugin/yaml/tests/sample_plugin.yaml‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,11 @@ pluginmods:
66
- dependencyresolver
77
pluginversion: latest
88
image: docker.io/odpf/optimus-task-bq2bq-executor:latest
9-
entrypoint: "sleep 100"
9+
entrypoint:
10+
shell: "/bin/bash"
11+
script: |-
12+
sleep 100
13+
sleep 150
1014
1115
questions:
1216
- name: PROJECT

‎plugin/yaml/tests/sample_plugin_without_shell.yaml‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
name: bq2bqtest
2+
description: Testing
3+
plugintype: task
4+
pluginmods:
5+
- cli
6+
- dependencyresolver
7+
pluginversion: latest
8+
image: docker.io/odpf/optimus-task-bq2bq-executor:latest
9+
entrypoint:
10+
script: "sleep 100"
11+
12+
questions:
13+
- name: PROJECT
14+
prompt: Project ID
15+
regexp: ^[a-zA-Z0-9_\-]+$
16+
minlength: 3
17+
18+
defaultconfig:
19+
- name: TEST
20+
value: "{{.test}}"
21+
22+
defaultassets:
23+
- name: query.sql
24+
value: Select * from "project.dataset.table";

‎plugin/yaml/tests/sample_plugin_without_version.yaml‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,9 @@ pluginmods:
55
- cli
66
pluginversion: ""
77
image: ""
8-
entrypoint: "sleep 100"
8+
entrypoint:
9+
shell: "/bin/bash"
10+
script: "sleep 100"
911

1012
questions:
1113
- name: PROJECT

‎sdk/plugin/mock/plugin.go‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -34,8 +34,10 @@ func (p *MockYamlMod) PluginInfo() *plugin.Info {
3434
DependsOn: nil,
3535
HookType: "",
3636
Image: "gcr.io/bq-plugin:dev",
37-
Entrypoint: "sleep 60",
38-
PluginMods: []plugin.Mod{plugin.ModTypeCLI},
37+
Entrypoint: plugin.Entrypoint{
38+
Script: "sleep 60",
39+
},
40+
PluginMods: []plugin.Mod{plugin.ModTypeCLI},
3941
}
4042
}
4143

0 commit comments

Comments
 (0)