Skip to content

Commit 4ce33b5

Browse files
bjyoungbloodbjyoungblood
authored
Add make_precompiler_downloader option (#87)
Allows users to customize download behavior, such as adding HTTP authentication or using an alternate protocol like SFTP.
1 parent d12a4a6 commit 4ce33b5

File tree

7 files changed

+220
-142
lines changed

7 files changed

+220
-142
lines changed

‎lib/elixir_make/artefact.ex‎

Lines changed: 3 additions & 136 deletions
Original file line numberDiff line numberDiff line change
@@ -286,141 +286,8 @@ defmodule ElixirMake.Artefact do
286286
end
287287
end
288288

289-
## Download
290-
291-
def download(url) do
292-
url_charlist = String.to_charlist(url)
293-
294-
# TODO: Remove me when we require Elixir v1.15
295-
{:ok, _} = Application.ensure_all_started(:inets)
296-
{:ok, _} = Application.ensure_all_started(:ssl)
297-
{:ok, _} = Application.ensure_all_started(:public_key)
298-
299-
if proxy = System.get_env("HTTP_PROXY") || System.get_env("http_proxy") do
300-
Mix.shell().info("Using HTTP_PROXY: #{proxy}")
301-
%{host: host, port: port} = URI.parse(proxy)
302-
303-
:httpc.set_options([{:proxy, {{String.to_charlist(host), port}, []}}])
304-
end
305-
306-
if proxy = System.get_env("HTTPS_PROXY") || System.get_env("https_proxy") do
307-
Mix.shell().info("Using HTTPS_PROXY: #{proxy}")
308-
%{host: host, port: port} = URI.parse(proxy)
309-
:httpc.set_options([{:https_proxy, {{String.to_charlist(host), port}, []}}])
310-
end
311-
312-
# https://erlef.github.io/security-wg/secure_coding_and_deployment_hardening/inets
313-
# TODO: This may no longer be necessary from Erlang/OTP 25.0 or later.
314-
https_options = [
315-
ssl:
316-
[
317-
verify: :verify_peer,
318-
customize_hostname_check: [
319-
match_fun: :public_key.pkix_verify_hostname_match_fun(:https)
320-
]
321-
] ++ cacerts_options()
322-
]
323-
324-
options = [body_format: :binary]
325-
326-
case :httpc.request(:get, {url_charlist, []}, https_options, options) do
327-
{:ok, {{_, 200, _}, _headers, body}} ->
328-
{:ok, body}
329-
330-
other ->
331-
{:error, "couldn't fetch NIF from #{url}: #{inspect(other)}"}
332-
end
333-
end
334-
335-
defp cacerts_options do
336-
cond do
337-
path = System.get_env("ELIXIR_MAKE_CACERT") ->
338-
[cacertfile: path]
339-
340-
certs = otp_cacerts() ->
341-
[cacerts: certs]
342-
343-
Application.spec(:castore, :vsn) ->
344-
[cacertfile: Application.app_dir(:castore, "priv/cacerts.pem")]
345-
346-
Application.spec(:certifi, :vsn) ->
347-
[cacertfile: Application.app_dir(:certifi, "priv/cacerts.pem")]
348-
349-
path = cacerts_from_os() ->
350-
[cacertfile: path]
351-
352-
true ->
353-
warn_no_cacerts()
354-
[]
355-
end
356-
end
357-
358-
defp otp_cacerts do
359-
if System.otp_release() >= "25" do
360-
# cacerts_get/0 raises if no certs found
361-
try do
362-
:public_key.cacerts_get()
363-
rescue
364-
_ ->
365-
nil
366-
end
367-
end
368-
end
369-
370-
# https_opts and related code are taken from
371-
# https://github.com/elixir-cldr/cldr_utils/blob/v2.19.1/lib/cldr/http/http.ex
372-
@certificate_locations [
373-
# Debian/Ubuntu/Gentoo etc.
374-
"/etc/ssl/certs/ca-certificates.crt",
375-
376-
# Fedora/RHEL 6
377-
"/etc/pki/tls/certs/ca-bundle.crt",
378-
379-
# OpenSUSE
380-
"/etc/ssl/ca-bundle.pem",
381-
382-
# OpenELEC
383-
"/etc/pki/tls/cacert.pem",
384-
385-
# CentOS/RHEL 7
386-
"/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem",
387-
388-
# Open SSL on MacOS
389-
"/usr/local/etc/openssl/cert.pem",
390-
391-
# MacOS & Alpine Linux
392-
"/etc/ssl/cert.pem"
393-
]
394-
395-
defp cacerts_from_os do
396-
Enum.find(@certificate_locations, &File.exists?/1)
397-
end
398-
399-
defp warn_no_cacerts do
400-
Mix.shell().error("""
401-
No certificate trust store was found.
402-
403-
Tried looking for: #{inspect(@certificate_locations)}
404-
405-
A certificate trust store is required in
406-
order to download locales for your configuration.
407-
Since elixir_make could not detect a system
408-
installed certificate trust store one of the
409-
following actions may be taken:
410-
411-
1. Install the hex package `castore`. It will
412-
be automatically detected after recompilation.
413-
414-
2. Install the hex package `certifi`. It will
415-
be automatically detected after recompilation.
416-
417-
3. Specify the location of a certificate trust store
418-
by configuring it in environment variable:
419-
420-
export ELIXIR_MAKE_CACERT="/path/to/cacerts.pem"
421-
422-
4. Use OTP 25+ on an OS that has built-in certificate
423-
trust store.
424-
""")
289+
def download(config, url) do
290+
downloader = config[:make_precompiler_downloader] || ElixirMake.Downloader.Httpc
291+
downloader.download(url)
425292
end
426293
end

‎lib/elixir_make/downloader.ex‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
defmodule ElixirMake.Downloader do
2+
@moduledoc """
3+
The behaviour for downloader modules.
4+
"""
5+
6+
@doc """
7+
This callback should download the artefact from the given URL.
8+
"""
9+
@callback download(url :: String.t()) :: {:ok, iolist() | binary()} | {:error, String.t()}
10+
end

‎lib/elixir_make/downloader/httpc.ex‎

Lines changed: 142 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,142 @@
1+
defmodule ElixirMake.Downloader.Httpc do
2+
@moduledoc false
3+
4+
@behaviour ElixirMake.Downloader
5+
6+
@impl ElixirMake.Downloader
7+
def download(url) do
8+
url_charlist = String.to_charlist(url)
9+
10+
# TODO: Remove me when we require Elixir v1.15
11+
{:ok, _} = Application.ensure_all_started(:inets)
12+
{:ok, _} = Application.ensure_all_started(:ssl)
13+
{:ok, _} = Application.ensure_all_started(:public_key)
14+
15+
if proxy = System.get_env("HTTP_PROXY") || System.get_env("http_proxy") do
16+
Mix.shell().info("Using HTTP_PROXY: #{proxy}")
17+
%{host: host, port: port} = URI.parse(proxy)
18+
19+
:httpc.set_options([{:proxy, {{String.to_charlist(host), port}, []}}])
20+
end
21+
22+
if proxy = System.get_env("HTTPS_PROXY") || System.get_env("https_proxy") do
23+
Mix.shell().info("Using HTTPS_PROXY: #{proxy}")
24+
%{host: host, port: port} = URI.parse(proxy)
25+
:httpc.set_options([{:https_proxy, {{String.to_charlist(host), port}, []}}])
26+
end
27+
28+
# https://erlef.github.io/security-wg/secure_coding_and_deployment_hardening/inets
29+
# TODO: This may no longer be necessary from Erlang/OTP 25.0 or later.
30+
https_options = [
31+
ssl:
32+
[
33+
verify: :verify_peer,
34+
customize_hostname_check: [
35+
match_fun: :public_key.pkix_verify_hostname_match_fun(:https)
36+
]
37+
] ++ cacerts_options()
38+
]
39+
40+
options = [body_format: :binary]
41+
42+
case :httpc.request(:get, {url_charlist, []}, https_options, options) do
43+
{:ok, {{_, 200, _}, _headers, body}} ->
44+
{:ok, body}
45+
46+
other ->
47+
{:error, "couldn't fetch NIF from #{url}: #{inspect(other)}"}
48+
end
49+
end
50+
51+
defp cacerts_options do
52+
cond do
53+
path = System.get_env("ELIXIR_MAKE_CACERT") ->
54+
[cacertfile: path]
55+
56+
certs = otp_cacerts() ->
57+
[cacerts: certs]
58+
59+
Application.spec(:castore, :vsn) ->
60+
[cacertfile: Application.app_dir(:castore, "priv/cacerts.pem")]
61+
62+
Application.spec(:certifi, :vsn) ->
63+
[cacertfile: Application.app_dir(:certifi, "priv/cacerts.pem")]
64+
65+
path = cacerts_from_os() ->
66+
[cacertfile: path]
67+
68+
true ->
69+
warn_no_cacerts()
70+
[]
71+
end
72+
end
73+
74+
defp otp_cacerts do
75+
if System.otp_release() >= "25" do
76+
# cacerts_get/0 raises if no certs found
77+
try do
78+
:public_key.cacerts_get()
79+
rescue
80+
_ ->
81+
nil
82+
end
83+
end
84+
end
85+
86+
# https_opts and related code are taken from
87+
# https://github.com/elixir-cldr/cldr_utils/blob/v2.19.1/lib/cldr/http/http.ex
88+
@certificate_locations [
89+
# Debian/Ubuntu/Gentoo etc.
90+
"/etc/ssl/certs/ca-certificates.crt",
91+
92+
# Fedora/RHEL 6
93+
"/etc/pki/tls/certs/ca-bundle.crt",
94+
95+
# OpenSUSE
96+
"/etc/ssl/ca-bundle.pem",
97+
98+
# OpenELEC
99+
"/etc/pki/tls/cacert.pem",
100+
101+
# CentOS/RHEL 7
102+
"/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem",
103+
104+
# Open SSL on MacOS
105+
"/usr/local/etc/openssl/cert.pem",
106+
107+
# MacOS & Alpine Linux
108+
"/etc/ssl/cert.pem"
109+
]
110+
111+
defp cacerts_from_os do
112+
Enum.find(@certificate_locations, &File.exists?/1)
113+
end
114+
115+
defp warn_no_cacerts do
116+
Mix.shell().error("""
117+
No certificate trust store was found.
118+
119+
Tried looking for: #{inspect(@certificate_locations)}
120+
121+
A certificate trust store is required in
122+
order to download locales for your configuration.
123+
Since elixir_make could not detect a system
124+
installed certificate trust store one of the
125+
following actions may be taken:
126+
127+
1. Install the hex package `castore`. It will
128+
be automatically detected after recompilation.
129+
130+
2. Install the hex package `certifi`. It will
131+
be automatically detected after recompilation.
132+
133+
3. Specify the location of a certificate trust store
134+
by configuring it in environment variable:
135+
136+
export ELIXIR_MAKE_CACERT="/path/to/cacerts.pem"
137+
138+
4. Use OTP 25+ on an OS that has built-in certificate
139+
trust store.
140+
""")
141+
end
142+
end

‎lib/mix/tasks/compile.elixir_make.ex‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,11 @@ defmodule Mix.Tasks.Compile.ElixirMake do
7070
* `:make_precompiler_filename` - the filename of the compiled artefact
7171
without its extension. Defaults to the app name.
7272
73+
* `:make_precompiler_downloader` - a module implementing the `ElixirMake.Downloader`
74+
behaviour. You can use this to customize how the precompiled artefacts
75+
are downloaded, for example, to add HTTP authentication or to download
76+
from an SFTP server. The default implementation uses `:httpc`.
77+
7378
* `:make_force_build` - if build should be forced even if precompiled artefacts
7479
are available. Defaults to true if the app has a `-dev` version flag.
7580
@@ -219,7 +224,7 @@ defmodule Mix.Tasks.Compile.ElixirMake do
219224
unless File.exists?(archived_fullpath) do
220225
Mix.shell().info("Downloading precompiled NIF to #{archived_fullpath}")
221226

222-
with {:ok, archived_data} <- Artefact.download(url) do
227+
with {:ok, archived_data} <- Artefact.download(config, url) do
223228
File.mkdir_p(Path.dirname(archived_fullpath))
224229
File.write(archived_fullpath, archived_data)
225230
end

‎lib/mix/tasks/elixir_make.checksum.ex‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -84,7 +84,7 @@ defmodule Mix.Tasks.ElixirMake.Checksum do
8484
Mix.raise("you need to specify either \"--all\" or \"--only-local\" flags")
8585
end
8686

87-
artefacts = download_and_checksum_all(urls, options)
87+
artefacts = download_and_checksum_all(config, urls, options)
8888

8989
if Keyword.get(options, :print, false) do
9090
artefacts
@@ -97,7 +97,7 @@ defmodule Mix.Tasks.ElixirMake.Checksum do
9797
Artefact.write_checksums!(artefacts)
9898
end
9999

100-
defp download_and_checksum_all(urls, options) do
100+
defp download_and_checksum_all(config, urls, options) do
101101
ignore_unavailable? = Keyword.get(options, :ignore_unavailable, false)
102102

103103
tasks =
@@ -106,7 +106,7 @@ defmodule Mix.Tasks.ElixirMake.Checksum do
106106
fn {{_target, _nif_version}, url} ->
107107
checksum_algo = Artefact.checksum_algo()
108108
checksum_file_url = "#{url}.#{Atom.to_string(checksum_algo)}"
109-
artifact_checksum = Artefact.download(checksum_file_url)
109+
artifact_checksum = Artefact.download(config, checksum_file_url)
110110

111111
with {:ok, body} <- artifact_checksum,
112112
[checksum, basename] <- String.split(body, " ", trim: true) do
@@ -117,7 +117,7 @@ defmodule Mix.Tasks.ElixirMake.Checksum do
117117
checksum_algo: checksum_algo
118118
}}
119119
else
120-
_ -> {:download, url, Artefact.download(url)}
120+
_ -> {:download, url, Artefact.download(config, url)}
121121
end
122122
end,
123123
timeout: :infinity,

‎test/fixtures/my_app/mix.exs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ defmodule MyApp.Precompiler do
1010
@behaviour ElixirMake.Precompiler
1111

1212
@impl true
13-
def current_target, do: "target"
13+
def current_target, do: {:ok, "target"}
1414

1515
@impl true
1616
def all_supported_targets(_), do: ["target"]

0 commit comments

Comments
 (0)