Skip to content

Commit a7c9796

Browse files
chrisrink10chrisrink10
authored
Add the basilisp.process namespace (#1136)
Fixes #1108
1 parent 52f2c21 commit a7c9796

File tree

5 files changed

+495
-1
lines changed

5 files changed

+495
-1
lines changed

‎CHANGELOG.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
77
## [Unreleased]
88
### Added
99
* Added support for a subset of qualified method syntax introduced in Clojure 1.12 (#1109)
10+
* Added the `basilisp.process` namespace (#1108)
1011

1112
### Changed
1213
* The Custom Data Readers Loader will only now examine the top directory and up to its immediate subdirectories of each `sys.path` entry, instead of recursive descending into every subdirectory, improving start up performance (#1135)

‎Makefile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ DOCBUILDDIR = "./docs/_build"
33

44
.PHONY: clean-docs
55
clean-docs:
6-
@rm -rf ./docs/build
6+
@rm -rf ./docs/_build
77

88
.PHONY: docs
99
docs:

‎docs/api/process.rst

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
basilisp.process
2+
================
3+
4+
.. toctree::
5+
:maxdepth: 2
6+
:caption: Contents:
7+
8+
.. autonamespace:: basilisp.process
9+
:members:
10+
:undoc-members:
11+
:exclude-members: FileWrapper, SubprocessRedirectable, ->FileWrapper, is-file-like?, is-path-like?

‎src/basilisp/process.lpy

Lines changed: 333 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,333 @@
1+
(ns basilisp.process
2+
"An API for starting subprocesses which wraps Python's :external:py:mod:`subprocess`
3+
module.
4+
5+
The primary API function is :lpy:fn:`start` which starts a subprocess and returns
6+
the :external:py:class:`subprocess.Popen` instance. You can wait for the process
7+
to terminate using :lpy:fn:`exit-ref` or you can manipulate it directly. You can
8+
fetch the streams attached to the process using :lpy:fn:`stdin`, :lpy:fn:`stdout`,
9+
and :lpy:fn:`stderr`. This namespace also includes an extension function
10+
:lpy:fn:`communicate` which wraps :external:py:meth:`subprocess.Popen.communicate`.
11+
12+
This namespace also includes :lpy:fn:`exec` for starting a subprocess, waiting for
13+
its completion, and returning the stdout as a string.
14+
15+
.. note::
16+
17+
There are some minor differences between the Basilisp implementation and the source
18+
Clojure implementation. Because Python does not have the concept of an unopened
19+
``File`` object as Java does, standard streams can only be passed existing open
20+
file handles or paths. If a path is given, the Basilisp :lpy:fn:`from-file` and
21+
:lpy:fn:`to-file` functions allow specifying the options that the file at that path
22+
will be opened with (including encoding, binary mode, etc.).
23+
24+
In Clojure, as in the underlying ``java.lang.Process`` object, streams are always
25+
opened in binary mode and it is the responsibility of callers to encode and decode
26+
into strings as necessary. Python's ``subprocesss`` offers some flexibility to
27+
callers to specify string encoding for pipes by default, saving them the effort of
28+
manually encoding and decoding and this namespace extends the same convenience."
29+
(:require
30+
[basilisp.io :as bio]
31+
[basilisp.string :as str])
32+
(:import contextlib
33+
os
34+
pathlib
35+
subprocess))
36+
37+
(defprotocol SubprocessRedirectable
38+
(is-file-like? [f]
39+
"Return true if ``f`` is a file-like object: either an integer (assumed to be a
40+
file handle or a file-like object with a ``.fileno()`` method).
41+
42+
This function says nothing about whether or not it is a valid file object or handle.")
43+
(is-path-like? [o]
44+
"Return true if ``o`` is a path-like object: either a string,
45+
:external:py:class:`pathlib.Path`, or byte string."))
46+
47+
(extend-protocol SubprocessRedirectable
48+
python/object
49+
(is-file-like? [f]
50+
(and (python/hasattr f "fileno")
51+
(python/callable (.-fileno f))))
52+
(is-path-like? [o]
53+
false)
54+
55+
python/int
56+
(is-file-like? [_]
57+
true)
58+
(is-path-like? [_]
59+
false)
60+
61+
python/str
62+
(is-file-like? [_]
63+
false)
64+
(is-path-like? [_]
65+
true)
66+
67+
python/bytes
68+
(is-file-like? [_]
69+
false)
70+
(is-path-like? [_]
71+
true)
72+
73+
pathlib/Path
74+
(is-file-like? [_]
75+
false)
76+
(is-path-like? [_]
77+
true))
78+
79+
(defrecord FileWrapper [path opts]
80+
SubprocessRedirectable
81+
(is-file-like? [self]
82+
false)
83+
(is-path-like? [self]
84+
true))
85+
86+
(defn from-file
87+
"Return a file object suitable for use as the ``:in`` option for :lpy:fn:`start`.
88+
89+
Callers can specify additional ``opts``. Anything supported by
90+
:lpy:fn:`basilisp.io/writer` and :lpy:fn:`basilisp.io/output-stream` is generally
91+
supported here. The values of individual ``opts`` are not validated until a call to
92+
:lpy:fn:`start` or :lpy:fn:`exec`.
93+
94+
.. warning::
95+
96+
``opts`` may only be specified for path-like values. Providing options for
97+
existing file objects and file handles will raise an exception."
98+
[f & {:as opts}]
99+
(cond
100+
(is-file-like? f) (if opts
101+
(throw
102+
(ex-info "Cannot specify options for an open file"
103+
{:file f :opts opts}))
104+
f)
105+
(is-path-like? f) (do
106+
(when (str/includes? (:mode opts "") "r")
107+
(throw
108+
(ex-info "Cannot specify :in file in read mode"
109+
{:file f :opts opts})))
110+
(->FileWrapper f opts))
111+
:else (throw
112+
(ex-info "Expected a file-like or path-like object"
113+
{:file f :opts opts}))))
114+
115+
(defn to-file
116+
"Return a file object suitable for use as the ``:err`` and ``:out`` options for
117+
:lpy:fn:`start`.
118+
119+
Callers can specify additional ``opts``. Anything supported by
120+
:lpy:fn:`basilisp.io/reader` and :lpy:fn:`basilisp.io/input-stream` is generally
121+
supported here. The values of individual ``opts`` are not validated until a call
122+
to :lpy:fn:`start` or :lpy:fn:`exec`.
123+
124+
.. warning::
125+
126+
``opts`` may only be specified for path-like values. Providing options for
127+
existing file objects and file handles will raise an exception."
128+
[f & {:as opts}]
129+
(cond
130+
(is-file-like? f) (if opts
131+
(throw
132+
(ex-info "Cannot specify options for an open file"
133+
{:file f :opts opts}))
134+
f)
135+
(is-path-like? f) (do
136+
(when (str/includes? (:mode opts "") "w")
137+
(throw
138+
(ex-info "Cannot specify :out or :err file in write mode"
139+
{:file f :opts opts})))
140+
(->FileWrapper f opts))
141+
:else (throw
142+
(ex-info "Expected a file-like or path-like object"
143+
{:file f :opts opts}))))
144+
145+
(defn exit-ref
146+
"Given a :external:py:class:`subprocess.Popen` (such as the one returned by
147+
:lpy:fn:`start`), return a reference which can be used to wait (optionally with
148+
timeout) for the completion of the process."
149+
[process]
150+
(reify
151+
basilisp.lang.interfaces/IBlockingDeref
152+
(deref [_ & args]
153+
;; basilisp.lang.runtime.deref converts Clojure's ms into seconds for Python
154+
(let [[timeout-s timeout-val] args]
155+
(if timeout-s
156+
(try
157+
(.wait process ** :timeout timeout-s)
158+
(catch subprocess/TimeoutExpired _
159+
timeout-val))
160+
(.wait process))))))
161+
162+
(defn ^:private wrapped-file-context-manager
163+
"Wrap a potential file in a context manager for ``start``.
164+
165+
Existing file-objects we just pass through using a null context manager, but
166+
path-likes need to be opened with a context manager."
167+
[f is-writer?]
168+
(if (is-path-like? f)
169+
(let [path (:path f f)
170+
opts (:opts f)
171+
is-binary? (str/includes? (:mode opts "") "b")
172+
io-fn (if is-binary?
173+
(if is-writer?
174+
bio/output-stream
175+
bio/input-stream)
176+
(if is-writer?
177+
bio/writer
178+
bio/reader))]
179+
(->> (mapcat identity opts)
180+
(apply io-fn path)))
181+
(contextlib/nullcontext f)))
182+
183+
(defn start
184+
"Start an external command as ``args``.
185+
186+
If ``opts`` are specified, they should be provided as a map in the first argument
187+
position.
188+
189+
The following options are available:
190+
191+
:keyword ``:in``: an existing file object or file handle, ``:pipe`` to generate a
192+
new stream, or ``:inherit`` to use the current process stdin; if not specified
193+
``:pipe`` will be used
194+
:keyword ``:out``: an existing file object or file handle, ``:pipe`` to generate a
195+
new stream, ``:discard`` to ignore stdout, or ``:inherit`` to use the current
196+
process stdout; if not specified, ``:pipe`` will be used
197+
:keyword ``:err``: an existing file object or file handle, ``:pipe`` to generate a
198+
new stream, ``:discard`` to ignore stderr, ``:stdout`` to merge stdout and
199+
stderr, or ``:inherit`` to use the current process stderr; if not specified,
200+
``:pipe`` will be used
201+
:keyword ``:dir``: current directory when the process runs; on POSIX systems, if
202+
executable is a relative path, it will be resolved relative to this value;
203+
if not specified, the current directory will be used
204+
:keyword ``:clear-env``: boolean which if ``true`` will prevent inheriting the
205+
environment variables from the current process
206+
:keyword ``:env``: a mapping of string values to string values which are added to
207+
the subprocess environment; if ``:clear-env``, these are the only environment
208+
variables provided to the subprocess
209+
210+
The following options affect the pipe streams created by
211+
:external:py:class:`subprocess.Popen` (if ``:pipe`` is selected), but do not apply
212+
to any files wrapped by :lpy:fn:`from-file` or :lpy:fn:`to-file` (in which case the
213+
options provided for those files take precedence) or if ``:inherit`` is specified
214+
(in which case the options for the corresponding stream in the current process is
215+
used). These options are specific to Basilisp.
216+
217+
:keyword ``:encoding``: the string name of an encoding to use for input and output
218+
streams when ``:pipe`` is specified for any of the standard streams; this option
219+
does not apply to any files wrapped by :lpy:fn:`from-file` or :lpy:fn:`to-file`
220+
or if ``:inherit`` is specified; if not specified, streams are treated as bytes
221+
222+
Returns :external:py:class:`subprocess.Popen` instance."
223+
[& opts+args]
224+
(let [[opts command] (if (map? (first opts+args))
225+
[(first opts+args) (rest opts+args)]
226+
[nil opts+args])
227+
228+
{:keys [in out err dir encoding]
229+
:or {in :pipe out :pipe err :pipe dir "."}} opts
230+
231+
stdin (condp = in
232+
:pipe subprocess/PIPE
233+
:inherit nil
234+
in)
235+
stdout (condp = out
236+
:pipe subprocess/PIPE
237+
:inherit nil
238+
:discard subprocess/DEVNULL
239+
out)
240+
stderr (condp = err
241+
:pipe subprocess/PIPE
242+
:discard subprocess/DEVNULL
243+
:stdout subprocess/STDOUT
244+
:inherit nil
245+
err)
246+
env (if (:clear-env opts)
247+
(:env opts {})
248+
(-> (python/dict os/environ)
249+
(py->lisp {:keywordize-keys false})
250+
(into (:env opts))))]
251+
;; Conditionally open files here if we're given a path-like since Python does
252+
;; not offer a `File` or `ProcessBuilder.Redirect` like object to handle this
253+
;; logic.
254+
(with [stdin (wrapped-file-context-manager stdin false)
255+
stdout (wrapped-file-context-manager stdout true)
256+
stderr (wrapped-file-context-manager stderr true)]
257+
(subprocess/Popen (python/list command)
258+
**
259+
:encoding encoding
260+
:stdin stdin
261+
:stdout stdout
262+
:stderr stderr
263+
:cwd (-> (pathlib/Path dir) (.resolve))
264+
:env (lisp->py env)))))
265+
266+
(defn exec
267+
"Execute a command as by :lpy:fn:`start` and, upon successful return, return the
268+
captured value of the process ``stdout`` as by :lpy:fn:`basilisp.core/slurp`.
269+
270+
If ``opts`` are specified, they should be provided as a map in the first argument
271+
position. ``opts`` are exactly the same as those in :lpy:fn:`start`.
272+
273+
If the return code is non-zero, throw
274+
:external:py:exc:`subprocess.CalledProcessError`."
275+
[& opts+args]
276+
(let [process (apply start opts+args)
277+
retcode (.wait process)]
278+
(if (zero? retcode)
279+
(slurp (.-stdout process))
280+
(throw
281+
(subprocess/CalledProcessError retcode
282+
(.-args process)
283+
(.-stdout process)
284+
(.-stderr process))))))
285+
286+
(defn stderr
287+
"Return the ``stderr`` stream from the external process.
288+
289+
.. warning::
290+
291+
Communication directly with the process streams introduces the possibility of
292+
deadlocks. Users may use :lpy:fn:`communicate` as a safe alternative."
293+
[process]
294+
(.-stderr process))
295+
296+
(defn stdin
297+
"Return the ``stdin`` stream from the external process.
298+
299+
.. warning::
300+
301+
Communication directly with the process streams introduces the possibility of
302+
deadlocks. Users may use :lpy:fn:`communicate` as a safe alternative."
303+
[process]
304+
(.-stdin process))
305+
306+
(defn stdout
307+
"Return the ``stdout`` stream from the external process.
308+
309+
.. warning::
310+
311+
Communication directly with the process streams introduces the possibility of
312+
deadlocks. Users may use :lpy:fn:`communicate` as a safe alternative."
313+
[process]
314+
(.-stdout process))
315+
316+
(defn communicate
317+
"Communicate with a subprocess, optionally sending data to the process stdin stream
318+
and reading any data in the process stderr and stdout streams, returning them as
319+
a string or bytes object (depending on whether the process was opened in text or
320+
binary mode).
321+
322+
This function is preferred over the use of :lpy:fn:`stderr`, :lpy:fn:`stdin`, and
323+
:lpy:fn:`stdout` to avoid potential deadlocks.
324+
325+
The following keyword/value arguments are optional:
326+
327+
:keyword ``:input``: a string or bytes object (depending on whether the process
328+
was opened in text or binary mode); if omitted, do not send anything
329+
:keyword ``timeout``: an optional timeout"
330+
[process & kwargs]
331+
(let [kwargs (apply hash-map kwargs)]
332+
(vec
333+
(.communicate process ** :input (:input kwargs) :timeout (:timeout kwargs)))))

0 commit comments

Comments
 (0)