Version 0.8.0: add sqlite / mount uenv repo images (#43)

* add sqlite support

Author: simonpintarelli

.clangd

@@ -1,2 +1,2 @@
 CompileFlags:
-  Add: [-std=c++17]
+  Add: [-std=c++17, -Isrc]

VERSION

@@ -1 +1 @@
-0.7.0
+0.8.0

ci/Dockerfile

@@ -4,6 +4,8 @@ FROM ${DOCKER_CONTAINER}
 RUN zypper install -y\
     fuse3-devel\
     git \
+    sqlite3 \
+    sqlite3-devel \
     libopenssl-devel\
     util-linux\
     util-linux-systemd\

ci/tests/index.db.txt

@@ -0,0 +1,56 @@
+PRAGMA foreign_keys=OFF;
+BEGIN TRANSACTION;
+CREATE TABLE images (
+    sha256 TEXT PRIMARY KEY CHECK(length(sha256)==64),
+    id TEXT UNIQUE CHECK(length(id)==16),
+    date TEXT NOT NULL,
+    size INTEGER NOT NULL,
+    uarch TEXT NOT NULL,
+    system TEXT NOT NULL
+);
+INSERT INTO images VALUES('3e8f96370a4685a7413d344d98f69889c0ba6bb1d6c2d3d19ce01b6079c58c68','3e8f96370a4685a7','2024-03-11 17:08:35.976000+00:00',8881353294,'x86_64','santis');
+INSERT INTO images VALUES('4e8f96370a4685a7413d344d98f69889c0ba6bb1d6c2d3d19ce01b6079c58c68','4e8f96370a4685a7','2024-03-11 17:08:35.976000+00:00',8881353294,'x86_64','santis');
+INSERT INTO images VALUES('1736b4bb5ad9b3c5cae8878c71782a8bf2f2f739dbce8e039b629de418cb4dab','1736b4bb5ad9b3c5','2024-02-19 06:33:57.442000+00:00',3987993166,'x86_64','santis');
+CREATE TABLE uenv (
+    version_id INTEGER PRIMARY KEY,
+    name TEXT NOT NULL,
+    version TEXT NOT NULL,
+    UNIQUE (name, version)
+);
+INSERT INTO uenv VALUES(1,'icon-wcp','v1');
+INSERT INTO uenv VALUES(2,'prgenv-gnu','24.2');
+INSERT INTO uenv VALUES(3,'prgenv-gnu','24.3');
+CREATE TABLE tags (
+    version_id INTEGER,
+    tag TEXT NOT NULL,
+    sha256 TEXT NOT NULL,
+    PRIMARY KEY (version_id, tag),
+    FOREIGN KEY (version_id)
+        REFERENCES uenv (version_id)
+            ON DELETE CASCADE
+            ON UPDATE CASCADE,
+    FOREIGN KEY (sha256)
+        REFERENCES images (sha256)
+            ON DELETE CASCADE
+            ON UPDATE CASCADE
+);
+INSERT INTO tags VALUES(1,'latest','3e8f96370a4685a7413d344d98f69889c0ba6bb1d6c2d3d19ce01b6079c58c68');
+INSERT INTO tags VALUES(1,'v3','3e8f96370a4685a7413d344d98f69889c0ba6bb1d6c2d3d19ce01b6079c58c68');
+INSERT INTO tags VALUES(2,'latest','1736b4bb5ad9b3c5cae8878c71782a8bf2f2f739dbce8e039b629de418cb4dab');
+INSERT INTO tags VALUES(2,'v2','1736b4bb5ad9b3c5cae8878c71782a8bf2f2f739dbce8e039b629de418cb4dab');
+INSERT INTO tags VALUES(3,'v3','4e8f96370a4685a7413d344d98f69889c0ba6bb1d6c2d3d19ce01b6079c58c68');
+CREATE VIEW records AS
+SELECT
+    images.system  AS system,
+    images.uarch   AS uarch,
+    uenv.name      AS name,
+    uenv.version   AS version,
+    tags.tag       AS tag,
+    images.date    AS date,
+    images.size    AS size,
+    tags.sha256    AS sha256,
+    images.id      AS id
+FROM tags
+    INNER JOIN uenv   ON uenv.version_id = tags.version_id
+    INNER JOIN images ON images.sha256   = tags.sha256;
+COMMIT;

ci/tests/test_sqlite.bats

@@ -0,0 +1,60 @@
+function setup() {
+    bats_load_library bats-support
+    bats_load_library bats-assert
+    load ./common
+    SQFSDIR=$(mktemp -d)
+    export SQFSDIR
+    rm -f index.db
+    sqlite3 ${SQFSDIR}/index.db < ci/tests/index.db.txt
+
+    img_hashes=(1736b4bb5ad9b3c5cae8878c71782a8bf2f2f739dbce8e039b629de418cb4dab 3313739553fe6553f789a35325eb6954a37a7b85cdeab943d0878a05edaac998 3e8f96370a4685a7413d344d98f69889c0ba6bb1d6c2d3d19ce01b6079c58c68)
+
+    for hash in ${img_hashes[@]}
+    do
+        mkdir -p ${SQFSDIR}/images/$hash
+    done
+
+    dd=$(mktemp -d)
+    (
+        umask 022
+        cd "$dd" || exit
+        mkdir spack-install
+        echo "This is file A" >> spack-install/fileA.txt
+    )
+    mksquashfs "$dd"  ${SQFSDIR}/store.squashfs  -quiet -noappend && rm -r "$dd"
+
+    for hash in ${img_hashes[@]}
+    do
+        cp ${SQFSDIR}/store.squashfs  ${SQFSDIR}/images/${hash}/
+    done
+}
+
+function teardown() {
+    rm -r ${SQFSDIR}
+}
+
+
+@test "mount prgenv-gnu" {
+    export UENV_REPO_PATH=${SQFSDIR}
+    run_srun --uenv=prgenv-gnu/24.2 true
+    run_srun --uenv=prgenv-gnu:latest true
+    run_srun --uenv=prgenv-gnu/24.2:latest true
+}
+
+@test "mount jfrog image by id" {
+    export UENV_REPO_PATH=${SQFSDIR}
+    run_srun --uenv=1736b4bb5ad9b3c5 true
+}
+
+@test "mount jfrog image by sha256" {
+    export UENV_REPO_PATH=${SQFSDIR}
+    run_srun --uenv=1736b4bb5ad9b3c5cae8878c71782a8bf2f2f739dbce8e039b629de418cb4dab true
+}
+
+@test "attempt to mount ambiguous prgenv-gnu" {
+    export UENV_REPO_PATH=${SQFSDIR}
+    run_srun_unchecked --uenv=prgenv-gnu true
+    assert_output --partial 'More than one uenv matches.'
+}
+
+

docker/Dockerfile

@@ -28,7 +28,9 @@ RUN zypper install -y\
     util-linux-systemd\
     psmisc\
     psutils\
-    strace
+    strace \
+    sqlite3 \
+    sqlite3-devel
 
 # install python
 RUN curl -O https://www.python.org/ftp/python/3.10.11/Python-3.10.11.tgz \

docker/rebuild.sh

@@ -1,4 +1,4 @@
 #!/usr/bin/bash
 
 docker compose exec -w /slurm-uenv-mount slurm \
-    sh -c 'meson install -C /uenv-build'
+    sh -c "meson install -C /uenv-build"

docker/run-tests.sh

@@ -1,3 +1,3 @@
 #!/bin/bash
 
-bats /slurm-uenv-mount/ci/tests/test.bats
+bats /slurm-uenv-mount/ci/tests

external/catch2/LICENSE.txt

@@ -0,0 +1,23 @@
+Boost Software License - Version 1.0 - August 17th, 2003
+
+Permission is hereby granted, free of charge, to any person or organization
+obtaining a copy of the software and accompanying documentation covered by
+this license (the "Software") to use, reproduce, display, distribute,
+execute, and transmit the Software, and to prepare derivative works of the
+Software, and to permit third-parties to whom the Software is furnished to
+do so, all subject to the following:
+
+The copyright notices in the Software and this entire statement, including
+the above license grant, this restriction and the following disclaimer,
+must be included in all copies of the Software, in whole or in part, and
+all derivative works of the Software, unless such copies or derivative
+works are solely in the form of machine-executable object code generated by
+a source language processor.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.