Merge branch 'main' into mvle/jet-test-esm2-finetune

Author: nvmvle

.github/workflows/internal_tools.yml

@@ -27,7 +27,7 @@ repos:
     hooks:
       - id: license-header-check
         name: Run license-check script
-        entry: python internal/infra-bionemo/src/infra_bionemo/license_check.py -c scripts -c sub-packages -c docs -c internal --license-header ./license_header --modify
+        entry: python ci/scripts/license_check.py -c sub-packages -c docs -c ci --license-header ./license_header --modify
         language: python
         additional_dependencies: ["click==8.1.7"]
         pass_filenames: false

.pre-commit-config.yaml

@@ -64,7 +64,7 @@ EOF
 ## Drop this when pytorch images ship the fixed commit.
 ARG TE_TAG=9d4e11eaa508383e35b510dc338e58b09c30be73
 
-COPY ./patches/te.patch /tmp/te.patch
+COPY ./docker_build_patches/te.patch /tmp/te.patch
 RUN git clone --recurse-submodules https://github.com/NVIDIA/TransformerEngine.git /tmp/TransformerEngine && \
     cd /tmp/TransformerEngine && \
     git checkout --recurse-submodules ${TE_TAG} && \
@@ -354,7 +354,6 @@ FROM bionemo2-base AS release
 RUN mkdir -p /workspace/bionemo2/.cache/
 
 COPY VERSION .
-COPY ./scripts ./scripts
 COPY ./README.md ./
 # Copy over folders so that the image can run tests in a self-contained fashion.
 COPY ./ci/scripts ./ci/scripts

3rdparty/NeMo

@@ -15,8 +15,8 @@ script_args:
   # See jet/core/configs.py for the specification of the configuration class
   workspace: /workspace/bionemo2
   data_path: /data/evo2
-  artefacts_url: https://gitlab-master.nvidia.com/api/v4/projects/clara-discovery%2Fcuhyena/jobs/172596829/artifacts/dist/
-  file_name_wheel: cuhyena-0.1-cp312-cp312-linux_x86_64.whl
+  artefacts_url: https://__token__:${JET_GITLAB_TOKEN}@gitlab-master.nvidia.com/api/v4/projects/180496/packages/pypi/simple
+  file_name_wheel: cuhyena==v0.2.3+cuda12.9
   model: evo2
   variant: train
   config_name: 1b
@@ -40,8 +40,7 @@ script_args:
 script: |-
   INSTALL_FLAG="/tmp/install_done_${{SLURMD_NODENAME}}";
   if [ "$SLURM_LOCALID" = "0" ]; then
-      curl --location --header "PRIVATE-TOKEN:$JET_GITLAB_TOKEN" "${artefacts_url}/${file_name_wheel}" --output ${file_name_wheel}
-      pip install ${file_name_wheel}
+      pip install ${file_name_wheel} --index-url ${artefacts_url}
       touch $INSTALL_FLAG
   fi
   # All ranks wait until install flag file appears

Dockerfile

@@ -14,6 +14,24 @@
 # limitations under the License.
 
 
+#! /usr/bin/env python3
+
+# SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: LicenseRef-Apache2
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
 import ast
 import sys
 from dataclasses import dataclass

internal/Pypi_publish.md ci/Pypi_publish.mdinternal/Pypi_publish.md renamed to ci/Pypi_publish.md

@@ -5,12 +5,6 @@ set -xueo pipefail
 REPOSITORY_ROOT=$(git rev-parse --show-toplevel)
 cd $REPOSITORY_ROOT
 
-echo "Running ruff checks"
-set +e
-ruff check scripts/ sub-packages/ docs/
-E_RUFF_CHECK="$?"
-set -e
-
 echo "Running tach checks"
 set +e
 tach check
@@ -19,7 +13,7 @@ set -e
 
 echo "Running pre-commit checks"
 set +e
-pre-commit run --all-files --show-diff-on-failure --color always
+pre-commit run --all-files --show-diff-on-failure --color always --verbose
 E_PRE_COMMIT="$?"
 set -e
 
@@ -29,10 +23,6 @@ if [[ "${E_PRE_COMMIT}" != "0" ]]; then
     ANY_FAILURE=1
     echo "ERROR: pre-commit hooks failed! (exit: ${E_PRE_COMMIT})"
 fi
-if [[ "${E_RUFF_CHECK}" != "0" ]]; then
-  ANY_FAILURE=1
-  echo "ERROR: ruff check failed! (exit: ${E_RUFF_CHECK})"
-fi
 if [[ "${E_TACH_CHECK}" != "0" ]]; then
   ANY_FAILURE=1
   echo "ERROR: tach check failed! (exit: ${E_TACH_CHECK})"