Merge pull request #47 from rust-math/alt-env

Detect system MKL without pkg-config

Author: termoshtt

.github/workflows/intel-mkl-tool.yml

@@ -26,8 +26,9 @@ jobs:
   docker:
     runs-on: ubuntu-18.04
     strategy:
+      fail-fast: false
       matrix:
-        target: ["test", "seek", "package", "download"]
+        target: ["test", "package", "seek", "seek-ubuntu", "seek-centos"]
     steps:
     - uses: actions/checkout@v1
     - name: Test with mkl-rust container

.gitignore

@@ -1,7 +1,7 @@
 # Generated by Cargo
 # will have compiled files and executables
 target/
-.cargo/
+.cargo-*/
 
 # Remove Cargo.lock from gitignore if creating an executable, leave it for libraries
 # More information here http://doc.crates.io/guide.html#cargotoml-vs-cargolock

README.md

@@ -34,12 +34,25 @@ Please use `blas-sys`, `lapack-sys`, or `fftw-sys` to use BLAS, LAPACK, FFTW int
 fftw-sys = { version = "0.4", features = ["intel-mkl"] }
 ```
 
-## pkg-config
-
-This crate does not download archive if `pkg-config` finds MKL shared library installed by other way.
-Be sure to set `PKG_CONFIG_PATH` and `LD_LIBRARY_PATH` correctly.
-For debian and ubuntu users, [ci/Dockerfile](ci/Dockerfile) may be helpful.
-Windows is not supported yet.
+## Find system MKL libraries
+
+This crate will download archive from AWS S3 `rust-intel-mkl` bucket if no MKL library installed by other way, e.g. [apt]/[yum].
+`intel-mkl-src` seeks MKL in following manner:
+
+- Check `${OUT_DIR}` where previous build has downloaded
+- Seek using [pkg-config] crate
+  - `${PKG_CONFIG_PATH}` has to be set correctly. It may not be set by default in usual install.
+  - You can confirm it by checking the following command returns error.
+    ```
+    pkg-config --libs mkl-dynamic-lp64-iomp
+    ```
+- (experimental) Seek `${XDG_DATA_HOME}/intel-mkl-tool`
+- Seek a directory set by `${MKLROOT}` environment variable
+- Seek `/opt/intel/mkl`
+
+[apt]: https://software.intel.com/content/www/us/en/develop/articles/installing-intel-free-libs-and-python-apt-repo.html
+[yum]: https://software.intel.com/content/www/us/en/develop/articles/installing-intel-free-libs-and-python-yum-repo.html
+[pkg-config]: https://github.com/rust-lang/pkg-config-rs
 
 ## License
 MKL is distributed under the Intel Simplified Software License for Intel(R) Math Kernel Library, See [License.txt](License.txt).

ci/Dockerfile

@@ -1,6 +1,5 @@
 ARG RUST_VERSION
 FROM rust:${RUST_VERSION}
-ARG MKL_VERSION
 
 WORKDIR /mkl
 
@@ -10,10 +9,10 @@ RUN apt update \
  && rm -rf /var/lib/apt/lists/*
 
 COPY ./silent.cfg /mkl/
-RUN curl -sfLO http://registrationcenter-download.intel.com/akdlm/irc_nas/tec/16533/l_mkl_${MKL_VERSION}.tgz \
- && tar xf l_mkl_${MKL_VERSION}.tgz \
- && cd l_mkl_${MKL_VERSION}         \
- && ./install.sh -s ../silent.cfg   \
+RUN curl -sfLO http://registrationcenter-download.intel.com/akdlm/irc_nas/tec/16533/l_mkl_2020.1.217.tgz \
+ && tar xf l_mkl_2020.1.217.tgz   \
+ && cd l_mkl_2020.1.217           \
+ && ./install.sh -s ../silent.cfg \
  && rm -rf /mkl
 ENV PKG_CONFIG_PATH /opt/intel/mkl/bin/pkgconfig
 RUN sed -i "s/MKLROOT/prefix/g" ${PKG_CONFIG_PATH}/*.pc

ci/Makefile

@@ -1,18 +1,27 @@
-REGISTRY     := rustmath/mkl-rust
+REGISTRY     := rustmath
 RUST_VERSION := 1.43.0
-MKL_VERSION  := 2020.1.217
-IMAGE        := $(REGISTRY):$(RUST_VERSION)-$(MKL_VERSION)
 
-all: test
+all: test ubuntu centos
 
 build:
 	docker build .                               \
 		--build-arg "RUST_VERSION=$(RUST_VERSION)" \
-		--build-arg "MKL_VERSION=$(MKL_VERSION)"   \
-		-t $(IMAGE)
+		-t $(REGISTRY)/mkl-rust:$(RUST_VERSION)
 
-push: build
-	docker push $(IMAGE)
+push: build ubuntu centos
+	docker push $(REGISTRY)/mkl-rust:$(RUST_VERSION)
+	docker push $(REGISTRY)/mkl-centos:$(RUST_VERSION)
+	docker push $(REGISTRY)/mkl-ubuntu:$(RUST_VERSION)
 
 test: build
-	docker run -it --rm $(IMAGE) pkg-config --libs mkl-dynamic-lp64-iomp
+	docker run -it --rm $(REGISTRY)/mkl-rust:$(RUST_VERSION) pkg-config --libs mkl-dynamic-lp64-iomp
+
+ubuntu:
+	docker build . -f $@.Dockerfile              \
+		--build-arg "RUST_VERSION=$(RUST_VERSION)" \
+		-t $(REGISTRY)/mkl-ubuntu:$(RUST_VERSION)
+
+centos:
+	docker build . -f $@.Dockerfile              \
+		--build-arg "RUST_VERSION=$(RUST_VERSION)" \
+		-t $(REGISTRY)/mkl-centos:$(RUST_VERSION)

ci/centos.Dockerfile

@@ -0,0 +1,23 @@
+# CentOS with official yum install procedure
+
+FROM centos:7
+
+# Setup Intel MKL
+RUN yum-config-manager --add-repo https://yum.repos.intel.com/mkl/setup/intel-mkl.repo \
+ && rpm --import https://yum.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB \
+ && yum install -y intel-mkl-2020.0-088 gcc pkg-config openssl-devel \
+ && rm -rf /var/cache/yum/* \
+ && yum clean all
+
+# Setup Rust
+# From official setting in https://github.com/rust-lang/docker-rust
+ARG RUST_VERSION
+ENV RUSTUP_HOME=/usr/local/rustup
+ENV CARGO_HOME=/usr/local/cargo
+ENV PATH=/usr/local/cargo/bin:$PATH
+RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --no-modify-path --default-toolchain ${RUST_VERSION}
+# this may concern security issue for production use, but this container is designed for development use.
+RUN chmod -R a+w $RUSTUP_HOME $CARGO_HOME
+
+WORKDIR /src
+RUN chmod -R a+w /src

ci/ubuntu.Dockerfile

@@ -0,0 +1,36 @@
+# Ubuntu with official apt install procedure
+
+FROM ubuntu:18.04
+
+RUN apt update \
+ && apt install -y \
+      apt-utils \
+      curl \
+      gnupg \
+      libssl-dev \
+      pkg-config \
+ && apt-get clean \
+ && rm -rf /var/lib/apt/lists/*
+
+# Install MKL
+# https://software.intel.com/content/www/us/en/develop/articles/installing-intel-free-libs-and-python-apt-repo.html
+RUN curl -sfL https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB | apt-key add -
+RUN curl -sfL https://apt.repos.intel.com/setup/intelproducts.list -o /etc/apt/sources.list.d/intelproducts.list
+
+RUN apt update \
+ && apt install -y intel-mkl-2020.0.088 \
+ && apt-get clean \
+ && rm -rf /var/lib/apt/lists/*
+
+# Setup Rust
+# From official setting in https://github.com/rust-lang/docker-rust
+ARG RUST_VERSION
+ENV RUSTUP_HOME=/usr/local/rustup
+ENV CARGO_HOME=/usr/local/cargo
+ENV PATH=/usr/local/cargo/bin:$PATH
+RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --no-modify-path --default-toolchain ${RUST_VERSION}
+# this may concern security issue for production use, but this container is designed for development use.
+RUN chmod -R a+w $RUSTUP_HOME $CARGO_HOME
+
+WORKDIR /src
+RUN chmod -R a+w /src

intel-mkl-tool/Makefile

@@ -1,16 +1,18 @@
-HERE  := $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))
-IMAGE := rustmath/mkl-rust:1.43.0-2020.1.217
-
-DOCKER_OPTION := --rm -u $(shell id -u):$(shell id -g) -v $(HERE):/src --env CARGO_HOME=/src/.cargo
+HERE          := $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))
+RUST_VERSION  := 1.43.0
+DOCKER_OPTION := --rm -u $(shell id -u):$(shell id -g) -v $(HERE):/src
 
 test:
-	docker run $(DOCKER_OPTION) $(IMAGE) cargo test with_mkl -- --ignored
+	docker run $(DOCKER_OPTION) --env CARGO_HOME=/src/.cargo-cache rustmath/mkl-rust:$(RUST_VERSION) cargo test with_mkl -- --ignored
 
 seek:
-	docker run $(DOCKER_OPTION) $(IMAGE) cargo run --release -- seek
+	docker run $(DOCKER_OPTION) --env CARGO_HOME=/src/.cargo-cache rustmath/mkl-rust:$(RUST_VERSION) cargo run --release -- seek
 
 package:
-	docker run $(DOCKER_OPTION) $(IMAGE) cargo run --release -- package
+	docker run $(DOCKER_OPTION) --env CARGO_HOME=/src/.cargo-cache rustmath/mkl-rust:$(RUST_VERSION) cargo run --release -- package
+
+seek-ubuntu:
+	docker run $(DOCKER_OPTION) --env CARGO_HOME=/src/.cargo-ubuntu rustmath/mkl-ubuntu:$(RUST_VERSION) cargo run --release -- seek
 
-download:
-	docker run $(DOCKER_OPTION) $(IMAGE) cargo run --release -- download -o ./test_download
+seek-centos:
+	docker run $(DOCKER_OPTION) --env CARGO_HOME=/src/.cargo-centos rustmath/mkl-centos:$(RUST_VERSION) cargo run --release -- seek

intel-mkl-tool/src/cli.rs

@@ -58,6 +58,10 @@ fn main() -> Result<()> {
         }
 
         Opt::Seek {} => {
+            let available = Entry::available();
+            if available.is_empty() {
+                bail!("No MKL found");
+            }
             for lib in Entry::available() {
                 if let Ok(version) = lib.version() {
                     println!("{:<22}: {}.{}", lib.name(), version.0, version.1);

intel-mkl-tool/src/entry.rs

@@ -36,7 +36,8 @@ impl Targets {
         self.0.iter().any(|(_key, value)| value.is_some())
     }
 
-    fn seek(&mut self, dir: &Path) {
+    fn seek<P: AsRef<Path>>(&mut self, dir: P) {
+        let dir = dir.as_ref();
         for (key, value) in &mut self.0 {
             if dir.join(key).exists() {
                 value.get_or_insert(dir.canonicalize().unwrap());
@@ -95,10 +96,24 @@ impl Entry {
             }
         }
 
-        // XDG_DATA_HOME
+        // $XDG_DATA_HOME/intel-mkl-tool
         let path = xdg_home_path().join(config.name());
         targets.seek(&path);
 
+        // $MKLROOT
+        let mkl_root = std::env::var("MKLROOT").map(|path| PathBuf::from(path));
+        if let Ok(path) = mkl_root {
+            if path.exists() {
+                targets.seek(path.join("lib/intel64"));
+            }
+        }
+
+        // /opt/intel/mkl
+        let opt_mkl = PathBuf::from("/opt/intel/mkl");
+        if opt_mkl.exists() {
+            targets.seek(opt_mkl.join("lib/intel64"));
+        }
+
         if targets.found_any() {
             return Ok(Self { config, targets });
         } else {