Install Python packages in container

Author: jordancarlin

.devcontainer/Dockerfile

@@ -42,4 +42,11 @@ RUN apt-get clean autoclean
 RUN apt-get autoremove -y
 RUN rm -rf /var/lib/{apt,dpkg,cache,log}/*
 
+# Create Python virtual environment and install packages
+COPY requirements.txt /tmp/requirements.txt
+RUN /usr/bin/python3 -m venv /opt/venv && \
+    /opt/venv/bin/pip install -r /tmp/requirements.txt && \
+    rm /tmp/requirements.txt
+ENV PATH="/opt/venv/bin:$PATH"
+
 WORKDIR /workspace

Rakefile

@@ -119,7 +119,7 @@ namespace :test do
   # "Run the cross-validation against LLVM"
   task :llvm do
       begin
-        sh "#{$root}/.home/.venv/bin/python3 -m pytest ext/auto-inst/test_parsing.py -v"
+        sh "/opt/venv/bin/python3 -m pytest ext/auto-inst/test_parsing.py -v"
       rescue => e
         raise unless e.message.include?("status (5)") # don't fail on skipped tests
     end

backends/generators/tasks.rake

@@ -28,9 +28,9 @@ namespace :gen do
     inst_dir = cfg_arch.path / "inst"
     csr_dir = cfg_arch.path / "csr"
 
-    # Run the Go generator script using the same Python environment
+    # Run the Go generator script
     # Note: The script uses --output not --output-dir
-    sh "#{$root}/.home/.venv/bin/python3 #{$root}/backends/generators/Go/go_generator.py --inst-dir=#{inst_dir} --csr-dir=#{csr_dir} --output=#{output_dir}inst.go"
+    sh "/opt/venv/bin/python3 #{$root}/backends/generators/Go/go_generator.py --inst-dir=#{inst_dir} --csr-dir=#{csr_dir} --output=#{output_dir}inst.go"
   end
 
   desc <<~DESC
@@ -79,9 +79,9 @@ namespace :gen do
     resolved_codes_file.flush
 
     begin
-      # Run the C header generator script using the same Python environment
+      # Run the C header generator script
       # The script generates encoding.h for inclusion in C programs
-      sh "#{$root}/.home/.venv/bin/python3 #{$root}/backends/generators/c_header/generate_encoding.py --inst-dir=#{inst_dir} --csr-dir=#{csr_dir} --ext-dir=#{ext_dir} --resolved-codes=#{resolved_codes_file.path} --output=#{output_dir}encoding.out.h --include-all"
+      sh "/opt/venv/bin/python3 #{$root}/backends/generators/c_header/generate_encoding.py --inst-dir=#{inst_dir} --csr-dir=#{csr_dir} --ext-dir=#{ext_dir} --resolved-codes=#{resolved_codes_file.path} --output=#{output_dir}encoding.out.h --include-all"
     ensure
       resolved_codes_file.close
       resolved_codes_file.unlink

bin/pip

@@ -3,8 +3,4 @@
 ROOT=$(dirname $(realpath ${BASH_SOURCE[0]}))
 source $ROOT/setup
 
-if [ ! -v VIRTUAL_ENV ]; then
-  source ${ROOT}/.home/.venv/bin/activate
-fi
-
 $PIP "$@"

bin/setup

@@ -197,16 +197,6 @@ if [ ! -f $ROOT/ext/rbi-central/README.md ]; then
     git submodule update --init ext/rbi-central
 fi
 
-if [ ! -d $ROOT/.home/.venv ]; then
-    ${RUN} /usr/bin/python3 -m venv ${ROOT}/.home/.venv
-fi
-
-source ${ROOT}/.home/.venv/bin/activate
-
-if [ ! -f ${ROOT}/.home/.venv/bin/pre-commit ]; then
-    ${RUN} ${ROOT}/.home/.venv/bin/pip install -r requirements.txt
-fi
-
 # if [ ! -f $ROOT/ext/riscv-opcodes/README.md ]; then
 #   git submodule update --init ext/riscv-opcodes
 # fi
@@ -230,8 +220,8 @@ if [ "${CONTAINER_TYPE}" == "devcontainer" ]; then
     NPM="npm"
     NPX="npx"
     NODE="node"
-    PYTHON="${ROOT}/.home/.venv/bin/python3"
-    PIP="${ROOT}/.home/.venv/bin/pip"
+    PYTHON="/opt/venv/bin/python3"
+    PIP="/opt/venv/bin/pip"
     BASH="bash"
     GPP="g++"
     GDB="gdb"
@@ -245,8 +235,8 @@ elif [ "${CONTAINER_TYPE}" == "docker" -o "${CONTAINER_TYPE}" == "podman" ]; the
     NPM="${CONTAINER_BASE} npm"
     NPX="${CONTAINER_BASE} npx"
     NODE="${CONTAINER_BASE} node"
-    PYTHON="${CONTAINER_BASE} ${ROOT}/.home/.venv/bin/python3"
-    PIP="${CONTAINER_BASE} ${ROOT}/.home/.venv/bin/pip"
+    PYTHON="${CONTAINER_BASE} /opt/venv/bin/python3"
+    PIP="${CONTAINER_BASE} /opt/venv/bin/pip"
     BASH="${CONTAINER_BASE} bash"
     GPP="${CONTAINER_BASE} g++"
     GDB="${CONTAINER_BASE} gdb"
@@ -260,8 +250,8 @@ elif [ "${CONTAINER_TYPE}" == "singularity" ]; then
     NPM="singularity run ${HOME_OPT} ${CONTAINER_PATH} npm"
     NPX="singularity run ${HOME_OPT} ${CONTAINER_PATH} npx"
     NODE="singularity run ${HOME_OPT} ${CONTAINER_PATH} node"
-    PYTHON="singularity run ${HOME_OPT} ${CONTAINER_PATH} ${ROOT}/.home/.venv/bin/python3"
-    PIP="singularity run ${HOME_OPT} ${CONTAINER_PATH} ${ROOT}/.home/.venv/bin/pip"
+    PYTHON="singularity run ${HOME_OPT} ${CONTAINER_PATH} /opt/venv/bin/python3"
+    PIP="singularity run ${HOME_OPT} ${CONTAINER_PATH} /opt/venv/bin/pip"
     BASH="singularity run ${HOME_OPT} ${CONTAINER_PATH} bash"
     GPP="singularity run ${HOME_OPT} ${CONTAINER_PATH} g++"
     GDB="singularity run ${HOME_OPT} ${CONTAINER_PATH} gdb"

container.def

@@ -1,10 +1,8 @@
 Bootstrap: docker
 From: ubuntu:24.04
 
-# %files
-
-#   Gemfile $HOME/
-#   package.json $HOME/
+%files
+  requirements.txt /tmp/requirements.txt
 
 %post
   export DEBIAN_FRONTEND=noninteractive
@@ -46,8 +44,16 @@ From: ubuntu:24.04
   apt-get autoremove -y
   rm -rf /var/lib/{apt, dpkg, cache, log}
 
+  # Create Python virtual environment and install packages
+  /usr/bin/python3 -m venv /opt/venv
+  /opt/venv/bin/pip install -r /tmp/requirements.txt
+  rm /tmp/requirements.txt
+
   # cd $HOME
   # bundle install
 
   # npm i wavedrom-cli
   # npm i i -E  @antora/[email protected] @antora/[email protected] @antora/lunr-extension asciidoctor-kroki
+
+%environment
+  export PATH="/opt/venv/bin:$PATH"

tools/ruby-gems/udb/lib/udb/resolver.rb

@@ -148,7 +148,7 @@ def initialize(
       @gen_path = gen_path_override || (@repo_root / "gen")
       @std_path = std_path_override || (@repo_root / "spec" / "std" / "isa")
       @custom_path = custom_path_override || (@repo_root / "spec" / "custom" / "isa")
-      @python_path = python_path_override || (@repo_root / ".home" / ".venv" / "bin" / "python3")
+      @python_path = python_path_override || Pathname.new("/opt") / "venv" / "bin" / "python3"
 
       # cache of config names
       @cfg_info = T.let({}, T::Hash[T.any(String, Pathname), ConfigInfo])

tools/ruby-gems/udb/test/test_yaml_loader.rb

@@ -26,7 +26,7 @@ def resolve_yaml(yaml)
 
       stdout, stderr, status =
         Dir.chdir(Udb.repo_root) do
-          Open3.capture3("/bin/bash -c \"source #{Udb.repo_root}/.home/.venv/bin/activate && #{Udb.repo_root}/.home/.venv/bin/python3 #{UDB_GEM_PATH}/python/yaml_resolver.py resolve --no-progress --no-checks #{arch_dir} #{resolved_dir}\"")
+          Open3.capture3("/opt/venv/bin/python3 #{UDB_GEM_PATH}/python/yaml_resolver.py resolve --no-progress --no-checks #{arch_dir} #{resolved_dir}")
         end
       # puts stdout
       # puts stderr
@@ -53,7 +53,7 @@ def resolve_multi_yaml(*yamls)
         File.write(test_dir / "test#{i + 1}.yaml", yaml)
       end
 
-      system "/bin/bash -c \"source #{Udb.repo_root}/.home/.venv/bin/activate && #{Udb.repo_root}/.home/.venv/bin/python3 #{UDB_GEM_PATH}/python/yaml_resolver.py resolve --no-checks #{arch_dir} #{resolved_dir}\""
+      system "/opt/venv/bin/python3 #{UDB_GEM_PATH}/python/yaml_resolver.py resolve --no-checks #{arch_dir} #{resolved_dir}"
 
       if $CHILD_STATUS == 0
         YAML.load_file(resolved_dir / "test" / "test1.yaml")