Skip to content
This repository was archived by the owner on Jan 20, 2025. It is now read-only.

Commit 68c6f9d

Browse files
asottileasottile
authored
Merge pull request #202 from sebastian-echeverria/feature/heredocs
Feature/heredocs
2 parents 4ca0e8b + c33a0ef commit 68c6f9d

File tree

6 files changed

+242
-20
lines changed

6 files changed

+242
-20
lines changed

‎go.mod

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,6 @@ module github.com/asottile/dockerfile
33
go 1.16
44

55
require (
6-
github.com/moby/buildkit v0.9.0
7-
github.com/stretchr/testify v1.7.0
6+
github.com/moby/buildkit v0.12.5
7+
github.com/stretchr/testify v1.8.3
88
)

‎parse.go

Lines changed: 28 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -9,17 +9,25 @@ import (
99
"github.com/moby/buildkit/frontend/dockerfile/parser"
1010
)
1111

12+
// Represents info about a heredoc.
13+
type Heredoc struct {
14+
Name string
15+
FileDescriptor uint
16+
Content string
17+
}
18+
1219
// Represents a single line (layer) in a Dockerfile.
1320
// For example `FROM ubuntu:xenial`
1421
type Command struct {
15-
Cmd string // lowercased command name (ex: `from`)
16-
SubCmd string // for ONBUILD only this holds the sub-command
17-
Json bool // whether the value is written in json form
18-
Original string // The original source line
19-
StartLine int // The original source line number which starts this command
20-
EndLine int // The original source line number which ends this command
21-
Flags []string // Any flags such as `--from=...` for `COPY`.
22-
Value []string // The contents of the command (ex: `ubuntu:xenial`)
22+
Cmd string // lowercased command name (ex: `from`)
23+
SubCmd string // for ONBUILD only this holds the sub-command
24+
Json bool // whether the value is written in json form
25+
Original string // The original source line
26+
StartLine int // The original source line number which starts this command
27+
EndLine int // The original source line number which ends this command
28+
Flags []string // Any flags such as `--from=...` for `COPY`.
29+
Value []string // The contents of the command (ex: `ubuntu:xenial`)
30+
Heredocs []Heredoc // Extra heredoc content attachments
2331
}
2432

2533
// A failure in opening a file for reading.
@@ -78,6 +86,18 @@ func ParseReader(file io.Reader) ([]Command, error) {
7886
cmd.Value = append(cmd.Value, n.Value)
7987
}
8088

89+
if len(child.Heredocs) != 0 {
90+
// For heredocs, add heredocs extra lines to Original,
91+
// and to the heredocs list.
92+
cmd.Original = cmd.Original + "\n"
93+
for _, heredoc := range child.Heredocs {
94+
cmd.Original = cmd.Original + heredoc.Content + heredoc.Name + "\n"
95+
cmd.Heredocs = append(cmd.Heredocs, Heredoc{Name: heredoc.Name,
96+
FileDescriptor: heredoc.FileDescriptor,
97+
Content: heredoc.Content})
98+
}
99+
}
100+
81101
ret = append(ret, cmd)
82102
}
83103
return ret, nil

‎parse_test.go

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -116,3 +116,63 @@ func TestParseFile(t *testing.T) {
116116
}
117117
assert.Equal(t, expected, cmds)
118118
}
119+
120+
func TestParseReaderHeredocs(t *testing.T) {
121+
dockerfile := `RUN 3<<EOF
122+
source $HOME/.bashrc && echo $HOME
123+
echo "Hello" >> /hello
124+
echo "World!" >> /hello
125+
EOF
126+
`
127+
cmds, err := ParseReader(bytes.NewBufferString(dockerfile))
128+
assert.Nil(t, err)
129+
expected := []Command{
130+
Command{
131+
Cmd: "RUN",
132+
Original: dockerfile,
133+
StartLine: 1,
134+
EndLine: 5,
135+
Flags: []string{},
136+
Value: []string{"3<<EOF"},
137+
Heredocs: []Heredoc{
138+
Heredoc{
139+
Name: "EOF",
140+
FileDescriptor: 3,
141+
Content: "source $HOME/.bashrc && echo $HOME\necho \"Hello\" >> /hello\necho \"World!\" >> /hello\n"},
142+
},
143+
},
144+
}
145+
assert.Equal(t, expected, cmds)
146+
}
147+
148+
func TestParseReaderHeredocsMultiple(t *testing.T) {
149+
dockerfile := `COPY <<FILE1 <<FILE2 /dest
150+
content 1
151+
FILE1
152+
content 2
153+
FILE2
154+
`
155+
cmds, err := ParseReader(bytes.NewBufferString(dockerfile))
156+
assert.Nil(t, err)
157+
expected := []Command{
158+
Command{
159+
Cmd: "COPY",
160+
Original: dockerfile,
161+
StartLine: 1,
162+
EndLine: 5,
163+
Flags: []string{},
164+
Value: []string{"<<FILE1", "<<FILE2", "/dest"},
165+
Heredocs: []Heredoc{
166+
Heredoc{
167+
Name: "FILE1",
168+
FileDescriptor: 0,
169+
Content: "content 1\n"},
170+
Heredoc{
171+
Name: "FILE2",
172+
FileDescriptor: 0,
173+
Content: "content 2\n"},
174+
},
175+
},
176+
}
177+
assert.Equal(t, expected, cmds)
178+
}

‎pylib/main.go

Lines changed: 47 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,10 @@ package main
1010
// extern PyObject* PyDockerfile_GoParseError;
1111
// extern PyObject* PyDockerfile_NewCommand(
1212
// PyObject*, PyObject*, PyObject*, PyObject*, PyObject*, PyObject*,
13-
// PyObject*, PyObject*
13+
// PyObject*, PyObject*, PyObject*
14+
// );
15+
// extern PyObject* PyDockerfile_NewHeredoc(
16+
// PyObject*, PyObject*, PyObject*
1417
// );
1518
import "C"
1619
import (
@@ -65,6 +68,40 @@ func sliceToTuple(strs []string) *C.PyObject {
6568
return ret
6669
}
6770

71+
func heredocsToPy(heredocs []dockerfile.Heredoc) *C.PyObject {
72+
var pyName, pyFileDescriptor, pyContent *C.PyObject
73+
var ret *C.PyObject
74+
decrefAll := func() {
75+
C.Py_DecRef(pyName)
76+
C.Py_DecRef(pyFileDescriptor)
77+
C.Py_DecRef(pyContent)
78+
C.Py_DecRef(ret)
79+
}
80+
81+
ret = C.PyTuple_New(C.Py_ssize_t(len(heredocs)))
82+
for i, heredoc := range heredocs {
83+
pyName := stringToPy(heredoc.Name)
84+
if pyName == nil {
85+
decrefAll()
86+
return nil
87+
}
88+
89+
pyFileDescriptor := C.PyLong_FromLong(C.long(heredoc.FileDescriptor))
90+
91+
pyContent := stringToPy(heredoc.Content)
92+
if pyContent == nil {
93+
decrefAll()
94+
return nil
95+
}
96+
97+
pyHeredoc := C.PyDockerfile_NewHeredoc(
98+
pyName, pyFileDescriptor, pyContent,
99+
)
100+
C.PyTuple_SetItem(ret, C.Py_ssize_t(i), pyHeredoc)
101+
}
102+
return ret
103+
}
104+
68105
func boolToInt(b bool) int {
69106
if b {
70107
return 1
@@ -74,7 +111,7 @@ func boolToInt(b bool) int {
74111
}
75112

76113
func cmdsToPy(cmds []dockerfile.Command) *C.PyObject {
77-
var pyCmd, pySubCmd, pyJson, pyOriginal, pyStartLine, pyEndLine, pyValue *C.PyObject
114+
var pyCmd, pySubCmd, pyJson, pyOriginal, pyStartLine, pyEndLine, pyValue, pyHeredocs *C.PyObject
78115
var pyFlags *C.PyObject
79116
var ret *C.PyObject
80117
decrefAll := func() {
@@ -86,6 +123,7 @@ func cmdsToPy(cmds []dockerfile.Command) *C.PyObject {
86123
C.Py_DecRef(pyEndLine)
87124
C.Py_DecRef(pyFlags)
88125
C.Py_DecRef(pyValue)
126+
C.Py_DecRef(pyHeredocs)
89127
C.Py_DecRef(ret)
90128
}
91129

@@ -126,8 +164,14 @@ func cmdsToPy(cmds []dockerfile.Command) *C.PyObject {
126164
return nil
127165
}
128166

167+
pyHeredocs = heredocsToPy(cmd.Heredocs)
168+
if pyHeredocs == nil {
169+
decrefAll()
170+
return nil
171+
}
172+
129173
pyCmd := C.PyDockerfile_NewCommand(
130-
pyCmd, pySubCmd, pyJson, pyOriginal, pyStartLine, pyEndLine, pyFlags, pyValue,
174+
pyCmd, pySubCmd, pyJson, pyOriginal, pyStartLine, pyEndLine, pyFlags, pyValue, pyHeredocs,
131175
)
132176
C.PyTuple_SetItem(ret, C.Py_ssize_t(i), pyCmd)
133177
}

‎pylib/support.c

Lines changed: 42 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -30,11 +30,26 @@ PyObject* PyDockerfile_NewCommand(
3030
PyObject* start_line,
3131
PyObject* end_line,
3232
PyObject* flags,
33-
PyObject* value
33+
PyObject* value,
34+
PyObject* heredocs
3435
) {
3536
return PyObject_CallFunction(
36-
PyDockerfile_Command, "OOOOOOOO",
37-
cmd, sub_cmd, json, original, start_line, end_line, flags, value
37+
PyDockerfile_Command, "OOOOOOOOO",
38+
cmd, sub_cmd, json, original, start_line, end_line, flags, value, heredocs
39+
);
40+
}
41+
42+
/* Heredoc namedtuple */
43+
PyObject* PyDockerfile_Heredoc;
44+
45+
PyObject* PyDockerfile_NewHeredoc(
46+
PyObject* name,
47+
PyObject* file_descriptor,
48+
PyObject* content
49+
) {
50+
return PyObject_CallFunction(
51+
PyDockerfile_Heredoc, "OOO",
52+
name, file_descriptor, content
3853
);
3954
}
4055

@@ -53,15 +68,35 @@ static PyObject* _setup_module(PyObject* module) {
5368
PyModule_AddObject(module, "GoParseError", PyDockerfile_GoParseError);
5469

5570
PyObject* collections = PyImport_ImportModule("collections");
56-
PyDockerfile_Command = PyObject_CallMethod(
57-
collections, "namedtuple", "ss",
58-
"Command", "cmd sub_cmd json original start_line end_line flags value"
59-
);
71+
72+
// Set up a Command namedtuple object, with empty default for heredocs substructure.
73+
PyObject *args = Py_BuildValue("ss", "Command", "cmd sub_cmd json original start_line end_line flags value heredocs");
74+
PyObject *kwargs = PyDict_New();
75+
PyObject* defaults = Py_BuildValue("(())");
76+
PyDict_SetItemString(kwargs, "defaults", defaults);
77+
PyObject *namedtuple_method = PyObject_GetAttrString(collections, "namedtuple");
78+
PyDockerfile_Command = PyObject_Call(namedtuple_method, args, kwargs);
79+
Py_DECREF(args);
80+
Py_DECREF(kwargs);
81+
Py_DECREF(defaults);
82+
Py_DECREF(namedtuple_method);
6083
PyObject_SetAttrString(
6184
PyDockerfile_Command, "__module__",
6285
PyObject_GetAttrString(module, "__name__")
6386
);
6487
PyModule_AddObject(module, "Command", PyDockerfile_Command);
88+
89+
// Set up a Heredoc namedtuple object.
90+
PyDockerfile_Heredoc = PyObject_CallMethod(
91+
collections, "namedtuple", "ss",
92+
"Heredoc", "name file_descriptor content"
93+
);
94+
PyObject_SetAttrString(
95+
PyDockerfile_Heredoc, "__module__",
96+
PyObject_GetAttrString(module, "__name__")
97+
);
98+
PyModule_AddObject(module, "Heredoc", PyDockerfile_Heredoc);
99+
65100
Py_XDECREF(collections);
66101
}
67102
return module;

‎tests/dockerfile_test.py

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -101,3 +101,66 @@ def test_parse_file_success():
101101
start_line=2, end_line=2, original='CMD ["echo", "hi"]',
102102
),
103103
)
104+
105+
106+
def test_heredoc_string_success():
107+
test_string = (
108+
'RUN 3<<EOF\n'
109+
'source $HOME/.bashrc && echo $HOME\n'
110+
'echo "Hello" >> /hello\n'
111+
'echo "World!" >> /hello\n'
112+
'EOF\n'
113+
)
114+
ret = dockerfile.parse_string(test_string)
115+
assert ret == (
116+
dockerfile.Command(
117+
cmd='RUN', sub_cmd=None, json=False, flags=(),
118+
value=(
119+
'3<<EOF',
120+
),
121+
start_line=1, end_line=5, original=test_string,
122+
heredocs=(
123+
dockerfile.Heredoc(
124+
name='EOF',
125+
content='source $HOME/.bashrc && echo $HOME\n'
126+
'echo "Hello" >> /hello\n'
127+
'echo "World!" >> /hello\n',
128+
file_descriptor=3,
129+
),
130+
),
131+
),
132+
)
133+
134+
135+
def test_heredoc_string_multiple_success():
136+
test_string = (
137+
'COPY <<FILE1 <<FILE2 /dest\n'
138+
'content 1\n'
139+
'FILE1\n'
140+
'content 2\n'
141+
'FILE2\n'
142+
)
143+
ret = dockerfile.parse_string(test_string)
144+
assert ret == (
145+
dockerfile.Command(
146+
cmd='COPY', sub_cmd=None, json=False, flags=(),
147+
value=(
148+
'<<FILE1',
149+
'<<FILE2',
150+
'/dest',
151+
),
152+
start_line=1, end_line=5, original=test_string,
153+
heredocs=(
154+
dockerfile.Heredoc(
155+
name='FILE1',
156+
content='content 1\n',
157+
file_descriptor=0,
158+
),
159+
dockerfile.Heredoc(
160+
name='FILE2',
161+
content='content 2\n',
162+
file_descriptor=0,
163+
),
164+
),
165+
),
166+
)

0 commit comments

Comments
 (0)