Skip to content

Commit 20e3e42

Browse files
vsochvsoch
committed
adding converter script
1 parent d52176f commit 20e3e42

File tree

4 files changed

+366
-13
lines changed

4 files changed

+366
-13
lines changed

‎singularity/build/converter.py‎

Lines changed: 340 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,340 @@
1+
'''
2+
3+
converted.py: Parse a Dockerfile into a Singularity spec file
4+
5+
Copyright (c) 2016, Vanessa Sochat. All rights reserved.
6+
7+
'''
8+
9+
import json
10+
import os
11+
import re
12+
import sys
13+
14+
from singularity.utils import (
15+
write_file,
16+
read_file
17+
)
18+
19+
from logman import logger
20+
import json
21+
22+
23+
# Parsing functions ---------------------------------------------------------------
24+
25+
def parse_env(env):
26+
'''parse_env will parse a Dockerfile ENV command to a singularity appropriate one
27+
eg: ENV PYTHONBUFFER 1 --> export PYTHONBUFFER=1
28+
::note This has to handle multiple exports per line. In the case of having an =,
29+
It could be that we have more than one pair of variables. If no equals, then
30+
we probably don't. See:
31+
see: https://docs.docker.com/engine/reference/builder/#/env
32+
'''
33+
# If the user has "=" then we can have more than one export per line
34+
exports = []
35+
name = None
36+
value = None
37+
if re.search("=",env):
38+
pieces = [p for p in re.split("( |\\\".*?\\\"|'.*?')", env) if p.strip()]
39+
while len(pieces) > 0:
40+
contender = pieces.pop(0)
41+
# If there is an equal, we've found a name
42+
if re.search("=",contender):
43+
if name != None:
44+
exports.append(join_env(name,value))
45+
name = contender
46+
value = None
47+
else:
48+
if value == None:
49+
value = contender
50+
else:
51+
value = "%s %s" %(value,contender)
52+
exports.append(join_env(name,value))
53+
# otherwise, the rule is one per line
54+
else:
55+
name,value = re.split(' ',env,1)
56+
exports = ["export %s=%s" %(name,value)]
57+
environment = []
58+
# Clean exports, make sure we aren't using
59+
for export in exports:
60+
export = export.strip('\n').replace('"',"").replace("'","")
61+
environment.append(export)
62+
export = 'echo "\n%s" >> /environment' %(export)
63+
environment.append(export)
64+
return "%s\n" %"\n".join(environment)
65+
66+
67+
def join_env(name,value):
68+
# If it's the end of the string, we don't want a space
69+
if re.search("=$",name):
70+
if value != None:
71+
return "export %s%s" %(name,value)
72+
else:
73+
return "export %s" %(name)
74+
if value != None:
75+
return "export %s %s" %(name,value)
76+
return "export %s" %(name)
77+
78+
79+
def parse_cmd(cmd):
80+
'''parse_cmd will parse a Dockerfile CMD command to a singularity appropriate one
81+
eg: CMD /code/run_uwsgi.sh --> /code/run_uwsgi.sh.
82+
'''
83+
return "%s" %(cmd)
84+
85+
86+
def parse_entry(cmd):
87+
'''parse_entry will parse a Dockerfile ENTRYPOINT command to a singularity appropriate one
88+
eg: ENTRYPOINT /code/run_uwsgi.sh --> exec /code/run_uwsgi.sh.
89+
'''
90+
return 'exec %s "$@"' %(cmd)
91+
92+
93+
def parse_copy(copy_str):
94+
'''parse_copy will copy a file from one location to another. This likely will need
95+
tweaking, as the files might need to be mounted from some location before adding to
96+
the image.
97+
'''
98+
return "cp %s" %(copy_str)
99+
100+
101+
def parse_http(url,destination):
102+
'''parse_http will get the filename of an http address, and return a statement
103+
to download it to some location
104+
'''
105+
file_name = os.path.basename(url)
106+
download_path = "%s/%s" %(destination,file_name)
107+
return "curl %s -o %s" %(url,download_path)
108+
109+
110+
def parse_targz(targz,destination):
111+
'''parse_targz will return a commnd to extract a targz file to a destination.
112+
'''
113+
return "tar -xzvf %s %s" %(targz,destination)
114+
115+
116+
def parse_zip(zipfile,destination):
117+
'''parse_zipfile will return a commnd to unzip a file to a destination.
118+
'''
119+
return "unzip %s %s" %(zipfile,destination)
120+
121+
def parse_comment(cmd):
122+
'''parse_comment simply returns the line as a comment.
123+
:param cmd: the comment
124+
'''
125+
return "# %s" %(cmd)
126+
127+
128+
def parse_maintainer(cmd):
129+
'''parse_maintainer will eventually save the maintainer as metadata.
130+
For now we return as comment.
131+
:param cmd: the maintainer line
132+
'''
133+
return parse_comment(cmd)
134+
135+
136+
def parse_add(add):
137+
'''parse_add will copy multiple files from one location to another. This likely will need
138+
tweaking, as the files might need to be mounted from some location before adding to
139+
the image. The add command is done for an entire directory.
140+
:param add: the command to parse
141+
'''
142+
# In the case that there are newlines or comments
143+
command,rest = add.split('\n',1)
144+
from_thing,to_thing = command.split(" ")
145+
146+
# People like to use dots for PWD.
147+
if from_thing == ".":
148+
from_thing = os.getcwd()
149+
if to_thing == ".":
150+
to_thing = os.getcwd()
151+
152+
# If it's a url or http address, then we need to use wget/curl to get it
153+
if re.search("^http",from_thing):
154+
result = parse_http(url=from_thing,
155+
destination=to_thing)
156+
157+
# If it's a tar.gz, then we are supposed to uncompress
158+
if re.search(".tar.gz$",from_thing):
159+
result = parse_targz(targz=from_thing,
160+
destination=to_thing)
161+
162+
# If it's .zip, then we are supposed to unzip it
163+
if re.search(".zip$",from_thing):
164+
result = parse_zip(zipfile=from_thing,
165+
destination=to_thing)
166+
167+
# Is from thing a directory or something else?
168+
if os.path.isdir(from_thing):
169+
result = "cp -R %s %s" %(from_thing,to_thing)
170+
else:
171+
result = "cp %s %s" %(from_thing,to_thing)
172+
return "%s\n%s" %(result,rest)
173+
174+
175+
def parse_workdir(workdir):
176+
'''parse_workdir will simply cd to the working directory
177+
'''
178+
return "cd %s" %(workdir)
179+
180+
181+
def get_mapping():
182+
'''get_mapping returns a dictionary mapping from a Dockerfile command to a Singularity
183+
build spec section. Note - this currently ignores lines that we don't know what to do with
184+
in the context of Singularity (eg, EXPOSE, LABEL, USER, VOLUME, STOPSIGNAL, escape,
185+
MAINTAINER)
186+
:: note
187+
each KEY of the mapping should be a command start in the Dockerfile (eg, RUN)
188+
for each corresponding value, there should be a dictionary with the following:
189+
190+
- section: the Singularity build file section to write the new command to
191+
- fun: any function to pass the output through before writing to the section (optional)
192+
- json: Boolean, if the section can optionally have json (eg a list)
193+
194+
I'm not sure the subtle differences between add and copy, other than copy doesn't support
195+
external files. It should suffice for our purposes (for now) to use the same function
196+
(parse_add) until evidence for a major difference is determined.
197+
'''
198+
# Docker : Singularity
199+
add_command = {"section": "%post","fun": parse_add, "json": True }
200+
copy_command = {"section": "%post", "fun": parse_add, "json": True }
201+
cmd_command = {"section": "%runscript", "fun": parse_cmd, "json": True }
202+
env_command = {"section": "%post", "fun": parse_env, "json": False }
203+
comment_command = {"section": "%post", "fun": parse_comment, "json": False }
204+
from_command = {"section": "From", "json": False }
205+
run_command = {"section": "%post", "json": True}
206+
workdir_command = {"section": "%post","fun": parse_workdir, "json": False }
207+
entry_command = {"section": "%post", "fun": parse_entry, "json": True }
208+
return {"ADD": add_command,
209+
"COPY":copy_command,
210+
"CMD":cmd_command,
211+
"ENTRYPOINT":entry_command,
212+
"ENV": env_command,
213+
"FROM": from_command,
214+
"RUN":run_command,
215+
"WORKDIR":workdir_command,
216+
"MAINTAINER":comment_command,
217+
"VOLUME":comment_command}
218+
219+
220+
221+
def dockerfile_to_singularity(dockerfile_path, output_dir=None):
222+
'''dockerfile_to_singularity will return a Singularity build file based on
223+
a provided Dockerfile. If output directory is not specified, the string
224+
will be returned. Otherwise, a file called Singularity will be written to
225+
output_dir
226+
:param dockerfile_path: the path to the Dockerfile
227+
:param output_dir: the output directory to write the Singularity file to
228+
'''
229+
if os.path.basename(dockerfile_path) == "Dockerfile":
230+
spec = read_file(dockerfile_path)
231+
# Use a common mapping
232+
mapping = get_mapping()
233+
# Put into dict of keys (section titles) and list of commands (values)
234+
sections = organize_sections(lines=spec,
235+
mapping=mapping)
236+
# We have to, by default, add the Docker bootstrap
237+
sections["bootstrap"] = ["docker"]
238+
# Put into one string based on "order" variable in mapping
239+
build_file = print_sections(sections=sections,
240+
mapping=mapping)
241+
if output_dir != None:
242+
write_file("%s/Singularity" %(output_dir),build_file)
243+
print("Singularity spec written to %s" %(output_dir))
244+
return build_file
245+
# If we make it here, something didn't work
246+
logger.error("Could not find %s, exiting.", dockerfile_path)
247+
return sys.exit(1)
248+
249+
250+
def organize_sections(lines,mapping=None):
251+
'''organize_sections will break apart lines from a Dockerfile, and put into
252+
appropriate Singularity sections.
253+
:param lines: the raw lines from the Dockerfile
254+
:mapping: a dictionary mapping Docker commands to Singularity sections
255+
'''
256+
if mapping == None:
257+
mapping = get_mapping()
258+
sections = dict()
259+
startre = "|".join(["^%s" %x for x in mapping.keys()])
260+
command = None
261+
name = None
262+
for l in range(0,len(lines)):
263+
line = lines[l]
264+
# If it's a newline or comment, just add it to post
265+
if line == "\n" or re.search("^#",line):
266+
sections = parse_section(name="%post",
267+
command=line,
268+
mapping=mapping,
269+
sections=sections)
270+
elif re.search(startre,line):
271+
# Parse the last section, and start over
272+
if command != None and name != None:
273+
sections = parse_section(name=name,
274+
command=command,
275+
mapping=mapping,
276+
sections=sections)
277+
name,command = line.split(" ",1)
278+
else:
279+
# We have a continuation of the last command or an empty line
280+
command = "%s\n%s" %(command,line)
281+
return sections
282+
283+
def parse_section(sections,name,command,mapping=None):
284+
'''parse_section will take a command that has lookup key "name" as a key in "mapping"
285+
and add a line to the list of each in sections that will be rendered into a Singularity
286+
build file.
287+
:param sections: the current sections, a dictionary of keys (singularity section titles)
288+
and a list of lines.
289+
:param name: the name of the section to add
290+
:param command: the command to parse:
291+
:param mapping: the mapping object to use
292+
'''
293+
if mapping == None:
294+
mapping = get_mapping()
295+
if name in mapping:
296+
build_section = mapping[name]['section']
297+
# Can the command potentially be json (a list?)
298+
if mapping[name]['json']:
299+
try:
300+
command = " ".join(json.loads(command))
301+
except:
302+
pass
303+
# Do we need to pass it through a function first?
304+
if 'fun' in mapping[name]:
305+
command = mapping[name]['fun'](command)
306+
# Add to our dictionary of sections!
307+
if build_section not in sections:
308+
sections[build_section] = [command]
309+
else:
310+
sections[build_section].append(command)
311+
return sections
312+
313+
314+
def print_sections(sections,mapping=None):
315+
'''print_sections will take a sections object (dict with section names and
316+
list of commands) and parse into a common string, to output to file or return
317+
to user.
318+
:param sections: output from organize_sections
319+
:mapping: a dictionary mapping Docker commands to Singularity sections
320+
'''
321+
if mapping == None:
322+
mapping = get_mapping()
323+
finished_spec = None
324+
ordering = ['bootstrap',"From","%runscript","%post"]
325+
for section in ordering:
326+
# Was the section found in the file?
327+
if section in sections:
328+
content = "".join(sections[section])
329+
# A single command, intended to go after a colon (yaml)
330+
if not re.search("^%",section):
331+
content = "%s:%s" %(section,content)
332+
else:
333+
# A list of things to join, after the section header
334+
content = "%s\n%s" %(section,content)
335+
# Are we adding the first line?
336+
if finished_spec == None:
337+
finished_spec = content
338+
else:
339+
finished_spec = "%s\n%s" %(finished_spec,content)
340+
return finished_spec

‎singularity/build/main.py‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,12 +15,15 @@
1515
package
1616
)
1717

18+
from singularity.build.converter import dockerfile_to_singularity
19+
1820
from singularity.build.utils import (
1921
get_singularity_version,
2022
stop_if_result_none,
2123
test_container
2224
)
2325

26+
2427
from singularity.utils import download_repo
2528
from singularity.analysis.classify import (
2629
get_tags,
@@ -89,6 +92,19 @@ def run_build(build_dir,params,verbose=True):
8992
passing_params = "/tmp/params.pkl"
9093
pickle.dump(params,open(passing_params,'wb'))
9194

95+
# If there is not a specfile, but is a Dockerfile, try building that
96+
if not os.path.exists(params['spec_file']) and os.path.exists('Dockerfile'):
97+
bot.logger.warning("Build file %s not found in repository",params['spec_file'])
98+
bot.logger.warning("Dockerfile found in repository, will attempt build.")
99+
dockerfile = dockerfile_to_singularity(dockerfile_path='Dockerfile',
100+
output_dir=build_dir)
101+
bot.logger.info("""\n
102+
--------------------------------------------------------------
103+
Dockerfile
104+
--------------------------------------------------------------
105+
\n%s""" %(dockerfile))
106+
107+
# Now look for spec file
92108
if os.path.exists(params['spec_file']):
93109
bot.logger.info("Found spec file %s in repository",params['spec_file'])
94110

0 commit comments

Comments
 (0)