Skip to content

Commit 671abbc

Browse files
termoshtttermoshtt
committed
Split archive feature
1 parent ba7f0a0 commit 671abbc

File tree

6 files changed

+129
-107
lines changed

6 files changed

+129
-107
lines changed

‎intel-mkl-tool/Cargo.toml

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -11,20 +11,23 @@ license = "MIT"
1111

1212
[features]
1313
default = ["cli"]
14-
cli = ["structopt"]
14+
archive = ["curl", "tar", "zstd"]
15+
cli = ["structopt", "archive"]
1516

1617
[dependencies]
1718
anyhow = "1.0.31"
18-
curl = "0.4.29"
1919
derive_more = "0.99.8"
2020
dirs = "2.0.2"
2121
glob = "0.3.0"
2222
pkg-config = "0.3.17"
23-
tar = "0.4.29"
24-
zstd = "0.5.3"
23+
24+
# archive
25+
curl = { version = "0.4.29", optional = true }
26+
tar = { version = "0.4.29", optional = true }
27+
zstd = { version = "0.5.3", optional = true }
2528

2629
# CLI
27-
structopt = { version = "0.3.15", optional = true }
30+
structopt = { version = "0.3.15", optional = true }
2831

2932
[dev-dependencies]
3033
paste = "0.1.17"

‎intel-mkl-tool/src/config.rs

Lines changed: 0 additions & 82 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,5 @@
11
use crate::*;
2-
use curl::easy::Easy;
32
use derive_more::*;
4-
use std::fs;
53

64
pub const VALID_CONFIGS: &[&str] = &[
75
"mkl-dynamic-ilp64-iomp",
@@ -146,41 +144,6 @@ impl Config {
146144
}
147145
}
148146
}
149-
150-
/// Download archive from AWS S3, and expand into `${out_dir}/*.so`
151-
pub fn download<P: AsRef<Path>>(&self, out_dir: P) -> Result<()> {
152-
let out_dir = out_dir.as_ref();
153-
if out_dir.exists() {
154-
fs::create_dir_all(&out_dir)?;
155-
}
156-
let data = read_from_url(&format!("{}/{}.tar.zst", s3_addr(), self.name()))?;
157-
let zstd = zstd::stream::read::Decoder::new(data.as_slice())?;
158-
let mut arc = tar::Archive::new(zstd);
159-
arc.unpack(&out_dir)?;
160-
Ok(())
161-
}
162-
}
163-
164-
/// Helper for download file from URL
165-
///
166-
/// - This function expands obtained data into memory space
167-
///
168-
fn read_from_url(url: &str) -> Result<Vec<u8>> {
169-
let mut data = Vec::new();
170-
let mut handle = Easy::new();
171-
handle.fail_on_error(true)?;
172-
handle.url(url)?;
173-
{
174-
let mut transfer = handle.transfer();
175-
transfer
176-
.write_function(|new_data| {
177-
data.extend_from_slice(new_data);
178-
Ok(new_data.len())
179-
})
180-
.unwrap();
181-
transfer.perform().unwrap();
182-
}
183-
Ok(data)
184147
}
185148

186149
#[cfg(test)]
@@ -220,49 +183,4 @@ mod tests {
220183
assert!(Config::from_str("mkl-static-lp64-omp").is_err());
221184
Ok(())
222185
}
223-
224-
macro_rules! impl_test_download {
225-
($name:expr) => {
226-
paste::item! {
227-
#[test]
228-
fn [<download_$name>]() -> Result<()> {
229-
let name = $name;
230-
let cfg = Config::from_str(name)?;
231-
cfg.download(format!("test_download/{}", name))?;
232-
Ok(())
233-
}
234-
}
235-
};
236-
}
237-
238-
#[cfg(target_os = "windows")]
239-
mod macos {
240-
use super::*;
241-
impl_test_download!("mkl-dynamic-lp64-seq");
242-
impl_test_download!("mkl-dynamic-ilp64-seq");
243-
impl_test_download!("mkl-static-lp64-seq");
244-
impl_test_download!("mkl-static-ilp64-seq");
245-
}
246-
247-
#[cfg(target_os = "macos")]
248-
mod macos {
249-
use super::*;
250-
impl_test_download!("mkl-dynamic-lp64-seq");
251-
impl_test_download!("mkl-dynamic-lp64-iomp");
252-
impl_test_download!("mkl-dynamic-ilp64-seq");
253-
impl_test_download!("mkl-dynamic-ilp64-iomp");
254-
}
255-
256-
#[cfg(target_os = "linux")]
257-
mod linux {
258-
use super::*;
259-
impl_test_download!("mkl-dynamic-lp64-seq");
260-
impl_test_download!("mkl-dynamic-lp64-iomp");
261-
impl_test_download!("mkl-dynamic-ilp64-seq");
262-
impl_test_download!("mkl-dynamic-ilp64-iomp");
263-
impl_test_download!("mkl-static-lp64-seq");
264-
impl_test_download!("mkl-static-lp64-iomp");
265-
impl_test_download!("mkl-static-ilp64-seq");
266-
impl_test_download!("mkl-static-ilp64-iomp");
267-
}
268186
}

‎intel-mkl-tool/src/download.rs

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
use crate::*;
2+
use curl::easy::Easy;
3+
use std::fs;
4+
5+
impl Config {
6+
/// Download archive from AWS S3, and expand into `${out_dir}/*.so`
7+
pub fn download<P: AsRef<Path>>(&self, out_dir: P) -> Result<()> {
8+
let out_dir = out_dir.as_ref();
9+
if out_dir.exists() {
10+
fs::create_dir_all(&out_dir)?;
11+
}
12+
let data = read_from_url(&format!("{}/{}.tar.zst", s3_addr(), self.name()))?;
13+
let zstd = zstd::stream::read::Decoder::new(data.as_slice())?;
14+
let mut arc = tar::Archive::new(zstd);
15+
arc.unpack(&out_dir)?;
16+
Ok(())
17+
}
18+
}
19+
20+
/// Helper for download file from URL
21+
///
22+
/// - This function expands obtained data into memory space
23+
///
24+
fn read_from_url(url: &str) -> Result<Vec<u8>> {
25+
let mut data = Vec::new();
26+
let mut handle = Easy::new();
27+
handle.fail_on_error(true)?;
28+
handle.url(url)?;
29+
{
30+
let mut transfer = handle.transfer();
31+
transfer
32+
.write_function(|new_data| {
33+
data.extend_from_slice(new_data);
34+
Ok(new_data.len())
35+
})
36+
.unwrap();
37+
transfer.perform().unwrap();
38+
}
39+
Ok(data)
40+
}
41+
42+
#[cfg(test)]
43+
mod tests {
44+
use super::*;
45+
46+
macro_rules! impl_test_download {
47+
($name:expr) => {
48+
paste::item! {
49+
#[test]
50+
fn [<download_$name>]() -> Result<()> {
51+
let name = $name;
52+
let cfg = Config::from_str(name)?;
53+
cfg.download(format!("test_download/{}", name))?;
54+
Ok(())
55+
}
56+
}
57+
};
58+
}
59+
60+
#[cfg(target_os = "windows")]
61+
mod macos {
62+
use super::*;
63+
impl_test_download!("mkl-dynamic-lp64-seq");
64+
impl_test_download!("mkl-dynamic-ilp64-seq");
65+
impl_test_download!("mkl-static-lp64-seq");
66+
impl_test_download!("mkl-static-ilp64-seq");
67+
}
68+
69+
#[cfg(target_os = "macos")]
70+
mod macos {
71+
use super::*;
72+
impl_test_download!("mkl-dynamic-lp64-seq");
73+
impl_test_download!("mkl-dynamic-lp64-iomp");
74+
impl_test_download!("mkl-dynamic-ilp64-seq");
75+
impl_test_download!("mkl-dynamic-ilp64-iomp");
76+
}
77+
78+
#[cfg(target_os = "linux")]
79+
mod linux {
80+
use super::*;
81+
impl_test_download!("mkl-dynamic-lp64-seq");
82+
impl_test_download!("mkl-dynamic-lp64-iomp");
83+
impl_test_download!("mkl-dynamic-ilp64-seq");
84+
impl_test_download!("mkl-dynamic-ilp64-iomp");
85+
impl_test_download!("mkl-static-lp64-seq");
86+
impl_test_download!("mkl-static-lp64-iomp");
87+
impl_test_download!("mkl-static-ilp64-seq");
88+
impl_test_download!("mkl-static-ilp64-iomp");
89+
}
90+
}

‎intel-mkl-tool/src/entry.rs

Lines changed: 0 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -193,26 +193,6 @@ impl Entry {
193193
bail!("Cannot determine MKL versions");
194194
}
195195

196-
pub fn package(&self, out_dir: &Path) -> Result<PathBuf> {
197-
fs::create_dir_all(out_dir)?;
198-
let out = out_dir.join(format!("{}.tar.zst", self.name()));
199-
if out.exists() {
200-
bail!("Output archive already exits: {}", out.display());
201-
}
202-
let f = fs::File::create(&out)?;
203-
let buf = io::BufWriter::new(f);
204-
let zstd = zstd::stream::write::Encoder::new(buf, 6)?;
205-
let mut ar = tar::Builder::new(zstd);
206-
ar.mode(tar::HeaderMode::Deterministic);
207-
for (path, name) in self.found_files() {
208-
let lib = path.join(&name);
209-
ar.append_path_with_name(lib, name)?;
210-
}
211-
let zstd = ar.into_inner()?;
212-
zstd.finish()?;
213-
Ok(out)
214-
}
215-
216196
pub fn print_cargo_metadata(&self) {
217197
let paths: HashSet<PathBuf> = self
218198
.found_files()

‎intel-mkl-tool/src/lib.rs

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -90,12 +90,19 @@
9090
//! |mkl_cdft_core | libmkl_cdft_core.so| libmkl_cdft_core.a|
9191
//!
9292
93+
#![cfg_attr(not(feature = "archive"), allow(dead_code))]
94+
9395
use anyhow::*;
9496
use std::path::*;
9597

9698
mod config;
9799
mod entry;
98100

101+
#[cfg(feature = "archive")]
102+
mod download;
103+
#[cfg(feature = "archive")]
104+
mod package;
105+
99106
pub use config::*;
100107
pub use entry::*;
101108

‎intel-mkl-tool/src/package.rs

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
use crate::*;
2+
use std::{fs, io};
3+
4+
impl Entry {
5+
pub fn package(&self, out_dir: &Path) -> Result<PathBuf> {
6+
fs::create_dir_all(out_dir)?;
7+
let out = out_dir.join(format!("{}.tar.zst", self.name()));
8+
if out.exists() {
9+
bail!("Output archive already exits: {}", out.display());
10+
}
11+
let f = fs::File::create(&out)?;
12+
let buf = io::BufWriter::new(f);
13+
let zstd = zstd::stream::write::Encoder::new(buf, 6)?;
14+
let mut ar = tar::Builder::new(zstd);
15+
ar.mode(tar::HeaderMode::Deterministic);
16+
for (path, name) in self.found_files() {
17+
let lib = path.join(&name);
18+
ar.append_path_with_name(lib, name)?;
19+
}
20+
let zstd = ar.into_inner()?;
21+
zstd.finish()?;
22+
Ok(out)
23+
}
24+
}

0 commit comments

Comments
 (0)