From 3bd432809d956f98f0f507f1419ea2616bf0a484 Mon Sep 17 00:00:00 2001 From: Alejandro Date: Fri, 11 Oct 2024 10:54:11 -0700 Subject: [PATCH 1/9] added reusable action for publishing validators to private pypi --- .../actions/validator_pypi_publish/action.yml | 77 +++++++++++++++++++ .../add_build_prefix.py | 37 +++++++++ 2 files changed, 114 insertions(+) create mode 100644 .github/actions/validator_pypi_publish/action.yml create mode 100644 .github/actions/validator_pypi_publish/add_build_prefix.py diff --git a/.github/actions/validator_pypi_publish/action.yml b/.github/actions/validator_pypi_publish/action.yml new file mode 100644 index 000000000..c1bcce809 --- /dev/null +++ b/.github/actions/validator_pypi_publish/action.yml @@ -0,0 +1,77 @@ +name: Publish Validator PyPi +description: Re-Usable action to publish a Validator to Guardrails PyPi +inputs: + github_token: + description: 'GitHub Token' + required: true + guardrails_token: + description: 'Guardrails Token' + required: true + pypi_repository_url: + description: 'PyPi Repository URL' + required: false + default: 'https://pypi.guardrailsai.com' + +runs: + using: "composite" + steps: + - name: Checkout "Validator" Repository + uses: actions/checkout@v3 + with: + path: 'validator' + + - name: Checkout "Action" repository + uses: actions/checkout@v3 + with: + token: ${{ inputs.github_token }} + repository: guardrails-ai/shared-ci + ref: main + path: shared-ci-scripts + sparse-checkout: | + .github + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.10' + + - name: Install Twine & Build + shell: bash + run: | + python -m pip install --upgrade pip + pip install twine build toml + + - name: Create .pypirc + shell: bash + run: | + touch ~/.pypirc + echo "[distutils]" >> ~/.pypirc + echo "index-servers =" >> ~/.pypirc + echo " private-repository" >> ~/.pypirc + echo "" >> ~/.pypirc + echo "[private-repository]" >> ~/.pypirc + echo "repository = ${{ inputs.pypi_repository_url }}" >> ~/.pypirc + echo "username = __token__" >> ~/.pypirc + echo "password = ${{ inputs.guardrails_token }}" >> ~/.pypirc + + - name: Move CI Scripts to Validator + shell: bash + run: | + mv shared-ci-scripts/.github/actions/validator_pypi_publish/add_build_prefix.py ./validator/__shared_ci_script__.py + + - name: Add Package Prefix + shell: bash + run: | + cd validator + python __shared_ci_script__.py ./pyproject.toml + echo "===== pyproject.toml file after edits ====" + cat ./pyproject.toml + echo "==========================================" + + - name: Build & Upload + shell: bash + run: | + cd validator + python -m build + twine upload dist/* -u __token__ -p ${{ inputs.guardrails_token }} -r private-repository + diff --git a/.github/actions/validator_pypi_publish/add_build_prefix.py b/.github/actions/validator_pypi_publish/add_build_prefix.py new file mode 100644 index 000000000..c502a4c0e --- /dev/null +++ b/.github/actions/validator_pypi_publish/add_build_prefix.py @@ -0,0 +1,37 @@ +import toml +import re +import sys + + +def add_package_name_prefix(pyproject_path): + with open(pyproject_path, "r") as f: + content = f.read() + + try: + toml_data = toml.loads(content) + except Exception as e: + print(f"Failed to parse {pyproject_path}: {e}") + sys.exit(1) + + existing_name = toml_data.get("project", {}).get("name") + if not existing_name: + print("Could not find the 'project.name' in pyproject.toml.") + sys.exit(1) + + new_name = f"guardrails-ai-validator-{existing_name}" + + updated_content = re.sub( + rf'(^name\s*=\s*")({re.escape(existing_name)})(")', + rf"\1{new_name}\3", + content, + flags=re.MULTILINE, + ) + + with open(pyproject_path, "w") as f: + f.write(updated_content) + + print(f"Updated project name to '{new_name}' in {pyproject_path}") + + +if __name__ == "__main__": + add_package_name_prefix("pyproject.toml") From e87116bc7d3176f1d8d497f51f32ee2ac5a43dc7 Mon Sep 17 00:00:00 2001 From: Alejandro Date: Fri, 11 Oct 2024 16:01:31 -0700 Subject: [PATCH 2/9] removed github token reference --- .github/actions/validator_pypi_publish/action.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/actions/validator_pypi_publish/action.yml b/.github/actions/validator_pypi_publish/action.yml index c1bcce809..cda3fbd52 100644 --- a/.github/actions/validator_pypi_publish/action.yml +++ b/.github/actions/validator_pypi_publish/action.yml @@ -1,9 +1,6 @@ name: Publish Validator PyPi description: Re-Usable action to publish a Validator to Guardrails PyPi inputs: - github_token: - description: 'GitHub Token' - required: true guardrails_token: description: 'Guardrails Token' required: true @@ -23,8 +20,7 @@ runs: - name: Checkout "Action" repository uses: actions/checkout@v3 with: - token: ${{ inputs.github_token }} - repository: guardrails-ai/shared-ci + repository: guardrails-ai/guardrails ref: main path: shared-ci-scripts sparse-checkout: | From 1a11aa2b4f5880956e6e901395bdb49827daf69c Mon Sep 17 00:00:00 2001 From: Alejandro Date: Wed, 16 Oct 2024 15:25:53 -0700 Subject: [PATCH 3/9] Added validator name input to publish npm action --- .../actions/validator_pypi_publish/action.yml | 8 ++++---- .../validator_pypi_publish/add_build_prefix.py | 16 ++++++++++++---- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/.github/actions/validator_pypi_publish/action.yml b/.github/actions/validator_pypi_publish/action.yml index cda3fbd52..56c2b28be 100644 --- a/.github/actions/validator_pypi_publish/action.yml +++ b/.github/actions/validator_pypi_publish/action.yml @@ -1,6 +1,9 @@ name: Publish Validator PyPi description: Re-Usable action to publish a Validator to Guardrails PyPi inputs: + validator: + description: 'Validator Name ex. guardrails/detect_pii' + required: true guardrails_token: description: 'Guardrails Token' required: true @@ -59,10 +62,7 @@ runs: shell: bash run: | cd validator - python __shared_ci_script__.py ./pyproject.toml - echo "===== pyproject.toml file after edits ====" - cat ./pyproject.toml - echo "==========================================" + python __shared_ci_script__.py ./pyproject.toml ${{ inputs.validator }} - name: Build & Upload shell: bash diff --git a/.github/actions/validator_pypi_publish/add_build_prefix.py b/.github/actions/validator_pypi_publish/add_build_prefix.py index c502a4c0e..f703c67f5 100644 --- a/.github/actions/validator_pypi_publish/add_build_prefix.py +++ b/.github/actions/validator_pypi_publish/add_build_prefix.py @@ -3,7 +3,7 @@ import sys -def add_package_name_prefix(pyproject_path): +def add_package_name_prefix(pyproject_path, validator_name): with open(pyproject_path, "r") as f: content = f.read() @@ -15,10 +15,11 @@ def add_package_name_prefix(pyproject_path): existing_name = toml_data.get("project", {}).get("name") if not existing_name: - print("Could not find the 'project.name' in pyproject.toml.") + print(f"Could not find the 'project.name' in {pyproject_path}.") sys.exit(1) - new_name = f"guardrails-ai-validator-{existing_name}" + validator_name = validator_name.split("/") + new_name = f"{validator_name[0]}-grhub-{validator_name[1]}" updated_content = re.sub( rf'(^name\s*=\s*")({re.escape(existing_name)})(")', @@ -34,4 +35,11 @@ def add_package_name_prefix(pyproject_path): if __name__ == "__main__": - add_package_name_prefix("pyproject.toml") + if len(sys.argv) < 3: + print("Usage: python script.py ") + sys.exit(1) + + pyproject_path = sys.argv[1] + validator_name = sys.argv[2] + + add_package_name_prefix(pyproject_path, validator_name) From ea0d27a5750e8e4c1fb1c33559bb0595d57926b9 Mon Sep 17 00:00:00 2001 From: Alejandro Date: Mon, 21 Oct 2024 16:51:02 -0400 Subject: [PATCH 4/9] Added additional scripts to shared action --- .../actions/validator_pypi_publish/action.yml | 9 ++++++--- .../validator_pypi_publish/concat_name.py | 16 ++++++++++++++++ .../package_name_normalization.py | 16 ++++++++++++++++ 3 files changed, 38 insertions(+), 3 deletions(-) create mode 100644 .github/actions/validator_pypi_publish/concat_name.py create mode 100644 .github/actions/validator_pypi_publish/package_name_normalization.py diff --git a/.github/actions/validator_pypi_publish/action.yml b/.github/actions/validator_pypi_publish/action.yml index 56c2b28be..756a6ccfd 100644 --- a/.github/actions/validator_pypi_publish/action.yml +++ b/.github/actions/validator_pypi_publish/action.yml @@ -56,13 +56,16 @@ runs: - name: Move CI Scripts to Validator shell: bash run: | - mv shared-ci-scripts/.github/actions/validator_pypi_publish/add_build_prefix.py ./validator/__shared_ci_script__.py + mv shared-ci-scripts/.github/actions/validator_pypi_publish/*.py ./validator - - name: Add Package Prefix + - name: Rename Package shell: bash run: | cd validator - python __shared_ci_script__.py ./pyproject.toml ${{ inputs.validator }} + CONCATANATED_NAME=$(python concat_name.py ${{ inputs.validator }}) + NEW_PEP_PACKAGE_NAME=$(python package_name_normalization.py $CONCATANATED_NAME) + python add_build_prefix.py ./pyproject.toml $CONCATANATED_NAME + mv ./validator ./$NEW_PEP_PACKAGE_NAME - name: Build & Upload shell: bash diff --git a/.github/actions/validator_pypi_publish/concat_name.py b/.github/actions/validator_pypi_publish/concat_name.py new file mode 100644 index 000000000..5c3de5fbc --- /dev/null +++ b/.github/actions/validator_pypi_publish/concat_name.py @@ -0,0 +1,16 @@ +def concat_name(validator_id): + validator_id_parts = validator_id.split("/") + namespace = validator_id_parts[0] + package_name = validator_id_parts[1] + return f"{namespace}-grhub-{package_name}" + + +if __name__ == "__main__": + import sys + + if len(sys.argv) < 2: + print("Usage: python concat_name.py ") + sys.exit(1) + + package_name = sys.argv[1] + print(concat_name(package_name)) diff --git a/.github/actions/validator_pypi_publish/package_name_normalization.py b/.github/actions/validator_pypi_publish/package_name_normalization.py new file mode 100644 index 000000000..43710e8ac --- /dev/null +++ b/.github/actions/validator_pypi_publish/package_name_normalization.py @@ -0,0 +1,16 @@ +from packaging.utils import canonicalize_name # PEP 503 + + +def normalize_package_name(concatanated_name: str) -> str: + return canonicalize_name(concatanated_name) + + +if __name__ == "__main__": + import sys + + if len(sys.argv) < 2: + print("Usage: python package_name_normalization.py ") + sys.exit(1) + + concatenated_name = sys.argv[1] + print(normalize_package_name(concatenated_name)) From 1d894a6d627d3c39d66e5579d33d308a5aa720a0 Mon Sep 17 00:00:00 2001 From: Alejandro Date: Mon, 21 Oct 2024 20:46:32 -0400 Subject: [PATCH 5/9] updated CI to inject setuptools section in pyproject toml --- .../actions/validator_pypi_publish/action.yml | 2 +- .../add_build_prefix.py | 60 +++++++++++++------ 2 files changed, 44 insertions(+), 18 deletions(-) diff --git a/.github/actions/validator_pypi_publish/action.yml b/.github/actions/validator_pypi_publish/action.yml index 756a6ccfd..27475346b 100644 --- a/.github/actions/validator_pypi_publish/action.yml +++ b/.github/actions/validator_pypi_publish/action.yml @@ -64,8 +64,8 @@ runs: cd validator CONCATANATED_NAME=$(python concat_name.py ${{ inputs.validator }}) NEW_PEP_PACKAGE_NAME=$(python package_name_normalization.py $CONCATANATED_NAME) - python add_build_prefix.py ./pyproject.toml $CONCATANATED_NAME mv ./validator ./$NEW_PEP_PACKAGE_NAME + python add_build_prefix.py ./pyproject.toml $NEW_PEP_PACKAGE_NAME - name: Build & Upload shell: bash diff --git a/.github/actions/validator_pypi_publish/add_build_prefix.py b/.github/actions/validator_pypi_publish/add_build_prefix.py index f703c67f5..00bccc546 100644 --- a/.github/actions/validator_pypi_publish/add_build_prefix.py +++ b/.github/actions/validator_pypi_publish/add_build_prefix.py @@ -1,45 +1,71 @@ -import toml import re import sys -def add_package_name_prefix(pyproject_path, validator_name): +def add_package_name_prefix(pyproject_path, pep_503_new_package_name): + # Read the existing pyproject.toml file with open(pyproject_path, "r") as f: content = f.read() try: - toml_data = toml.loads(content) + # Check for the presence of the project.name key using a regular expression + project_name_match = re.search( + r'^name\s*=\s*"(.*?)"', content, flags=re.MULTILINE + ) + if not project_name_match: + print(f"Could not find the 'project.name' in {pyproject_path}.") + sys.exit(1) + existing_name = project_name_match.group(1) except Exception as e: - print(f"Failed to parse {pyproject_path}: {e}") + print(f"Failed to parse project name in {pyproject_path}: {e}") sys.exit(1) - existing_name = toml_data.get("project", {}).get("name") - if not existing_name: - print(f"Could not find the 'project.name' in {pyproject_path}.") - sys.exit(1) - - validator_name = validator_name.split("/") - new_name = f"{validator_name[0]}-grhub-{validator_name[1]}" - + # Update the project name to the new PEP 503-compliant name updated_content = re.sub( rf'(^name\s*=\s*")({re.escape(existing_name)})(")', - rf"\1{new_name}\3", + rf"\1{pep_503_new_package_name}\3", content, flags=re.MULTILINE, ) + # Now we manually add the [tool.setuptools] section with the new folder name + # If the section already exists, we append the correct package name + setuptools_section = f""" + +[tool.setuptools] +packages = ["{pep_503_new_package_name}"] + +""" + + # Check if the [tool.setuptools] section already exists + if "[tool.setuptools]" in updated_content: + # If it exists, update the packages value + updated_content = re.sub( + r"(^\[tool\.setuptools\].*?^packages\s*=\s*\[.*?\])", + f'[tool.setuptools]\npackages = ["{pep_503_new_package_name}"]', + updated_content, + flags=re.DOTALL | re.MULTILINE, + ) + else: + # If the section doesn't exist, append it at the end of the file + updated_content += setuptools_section + + # Write the modified content back to the pyproject.toml file with open(pyproject_path, "w") as f: f.write(updated_content) - print(f"Updated project name to '{new_name}' in {pyproject_path}") + print( + "Updated project name to " + "'{pep_503_new_package_name}' and added package folder in {pyproject_path}" + ) if __name__ == "__main__": if len(sys.argv) < 3: - print("Usage: python script.py ") + print("Usage: python script.py ") sys.exit(1) pyproject_path = sys.argv[1] - validator_name = sys.argv[2] + pep_503_new_package_name = sys.argv[2] - add_package_name_prefix(pyproject_path, validator_name) + add_package_name_prefix(pyproject_path, pep_503_new_package_name) From 1622af42fa472944c0ba4db551abd7110e63e69c Mon Sep 17 00:00:00 2001 From: Alejandro Date: Mon, 21 Oct 2024 21:39:22 -0400 Subject: [PATCH 6/9] updated CI to match updated package bundling flow --- .../actions/validator_pypi_publish/action.yml | 5 ++-- .../add_build_prefix.py | 26 ++++++++++++------- 2 files changed, 20 insertions(+), 11 deletions(-) diff --git a/.github/actions/validator_pypi_publish/action.yml b/.github/actions/validator_pypi_publish/action.yml index 27475346b..f68dfe250 100644 --- a/.github/actions/validator_pypi_publish/action.yml +++ b/.github/actions/validator_pypi_publish/action.yml @@ -64,8 +64,9 @@ runs: cd validator CONCATANATED_NAME=$(python concat_name.py ${{ inputs.validator }}) NEW_PEP_PACKAGE_NAME=$(python package_name_normalization.py $CONCATANATED_NAME) - mv ./validator ./$NEW_PEP_PACKAGE_NAME - python add_build_prefix.py ./pyproject.toml $NEW_PEP_PACKAGE_NAME + VALIDATOR_FOLDER_NAME=$(echo $NEW_PEP_PACKAGE_NAME | tr - _) + mv ./validator ./$VALIDATOR_FOLDER_NAME + python add_build_prefix.py ./pyproject.toml $NEW_PEP_PACKAGE_NAME $VALIDATOR_FOLDER_NAME - name: Build & Upload shell: bash diff --git a/.github/actions/validator_pypi_publish/add_build_prefix.py b/.github/actions/validator_pypi_publish/add_build_prefix.py index 00bccc546..c77f60a24 100644 --- a/.github/actions/validator_pypi_publish/add_build_prefix.py +++ b/.github/actions/validator_pypi_publish/add_build_prefix.py @@ -2,7 +2,9 @@ import sys -def add_package_name_prefix(pyproject_path, pep_503_new_package_name): +def add_package_name_prefix( + pyproject_path, pep_503_new_package_name, validator_folder_name +): # Read the existing pyproject.toml file with open(pyproject_path, "r") as f: content = f.read() @@ -21,6 +23,8 @@ def add_package_name_prefix(pyproject_path, pep_503_new_package_name): sys.exit(1) # Update the project name to the new PEP 503-compliant name + # The package name would've been converted to PEP 503-compliant anyways + # But we use this name since it's been concatenated with the seperator updated_content = re.sub( rf'(^name\s*=\s*")({re.escape(existing_name)})(")', rf"\1{pep_503_new_package_name}\3", @@ -33,7 +37,7 @@ def add_package_name_prefix(pyproject_path, pep_503_new_package_name): setuptools_section = f""" [tool.setuptools] -packages = ["{pep_503_new_package_name}"] +packages = ["{validator_folder_name}"] """ @@ -42,7 +46,7 @@ def add_package_name_prefix(pyproject_path, pep_503_new_package_name): # If it exists, update the packages value updated_content = re.sub( r"(^\[tool\.setuptools\].*?^packages\s*=\s*\[.*?\])", - f'[tool.setuptools]\npackages = ["{pep_503_new_package_name}"]', + f'[tool.setuptools]\npackages = ["{validator_folder_name}"]', updated_content, flags=re.DOTALL | re.MULTILINE, ) @@ -54,18 +58,22 @@ def add_package_name_prefix(pyproject_path, pep_503_new_package_name): with open(pyproject_path, "w") as f: f.write(updated_content) - print( - "Updated project name to " - "'{pep_503_new_package_name}' and added package folder in {pyproject_path}" - ) + print(f"Updated project name to '{pep_503_new_package_name}'.") + print(f"Added package folder '{validator_folder_name}' in {pyproject_path}") if __name__ == "__main__": if len(sys.argv) < 3: - print("Usage: python script.py ") + print( + "Usage: python script.py " + " " + ) sys.exit(1) pyproject_path = sys.argv[1] pep_503_new_package_name = sys.argv[2] + validator_folder_name = sys.argv[3] - add_package_name_prefix(pyproject_path, pep_503_new_package_name) + add_package_name_prefix( + pyproject_path, pep_503_new_package_name, validator_folder_name + ) From e405dc655d5931f4f8882038b0b60a72505ad0eb Mon Sep 17 00:00:00 2001 From: Alejandro Date: Wed, 23 Oct 2024 19:44:03 -0700 Subject: [PATCH 7/9] changed shared ci inputs --- .github/actions/validator_pypi_publish/action.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/actions/validator_pypi_publish/action.yml b/.github/actions/validator_pypi_publish/action.yml index f68dfe250..37b353aef 100644 --- a/.github/actions/validator_pypi_publish/action.yml +++ b/.github/actions/validator_pypi_publish/action.yml @@ -1,8 +1,8 @@ name: Publish Validator PyPi description: Re-Usable action to publish a Validator to Guardrails PyPi inputs: - validator: - description: 'Validator Name ex. guardrails/detect_pii' + validator_id: + description: 'Validator ID ex. guardrails/detect_pii' required: true guardrails_token: description: 'Guardrails Token' @@ -62,7 +62,7 @@ runs: shell: bash run: | cd validator - CONCATANATED_NAME=$(python concat_name.py ${{ inputs.validator }}) + CONCATANATED_NAME=$(python concat_name.py ${{ inputs.validator_id }}) NEW_PEP_PACKAGE_NAME=$(python package_name_normalization.py $CONCATANATED_NAME) VALIDATOR_FOLDER_NAME=$(echo $NEW_PEP_PACKAGE_NAME | tr - _) mv ./validator ./$VALIDATOR_FOLDER_NAME From 008f8c3c48f608ee712f748b9989cd6190911d3f Mon Sep 17 00:00:00 2001 From: Alejandro Date: Wed, 23 Oct 2024 20:14:33 -0700 Subject: [PATCH 8/9] rename action --- .github/actions/validator_pypi_publish/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/validator_pypi_publish/action.yml b/.github/actions/validator_pypi_publish/action.yml index 37b353aef..81c0ab5fe 100644 --- a/.github/actions/validator_pypi_publish/action.yml +++ b/.github/actions/validator_pypi_publish/action.yml @@ -1,4 +1,4 @@ -name: Publish Validator PyPi +name: Publish to Guardrails Hub description: Re-Usable action to publish a Validator to Guardrails PyPi inputs: validator_id: From 91354fffeee5435109ad6bc13c5c61d220f854aa Mon Sep 17 00:00:00 2001 From: Alejandro Date: Thu, 24 Oct 2024 11:39:59 -0700 Subject: [PATCH 9/9] updated add build prefix file to parse name using toml --- .../add_build_prefix.py | 20 ++++++------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/.github/actions/validator_pypi_publish/add_build_prefix.py b/.github/actions/validator_pypi_publish/add_build_prefix.py index c77f60a24..d5069e551 100644 --- a/.github/actions/validator_pypi_publish/add_build_prefix.py +++ b/.github/actions/validator_pypi_publish/add_build_prefix.py @@ -1,5 +1,6 @@ import re import sys +import toml def add_package_name_prefix( @@ -8,19 +9,10 @@ def add_package_name_prefix( # Read the existing pyproject.toml file with open(pyproject_path, "r") as f: content = f.read() + parsed_toml = toml.loads(content) - try: - # Check for the presence of the project.name key using a regular expression - project_name_match = re.search( - r'^name\s*=\s*"(.*?)"', content, flags=re.MULTILINE - ) - if not project_name_match: - print(f"Could not find the 'project.name' in {pyproject_path}.") - sys.exit(1) - existing_name = project_name_match.group(1) - except Exception as e: - print(f"Failed to parse project name in {pyproject_path}: {e}") - sys.exit(1) + # get the existing package name + existing_name = parsed_toml.get("project", {}).get("name") # Update the project name to the new PEP 503-compliant name # The package name would've been converted to PEP 503-compliant anyways @@ -65,8 +57,8 @@ def add_package_name_prefix( if __name__ == "__main__": if len(sys.argv) < 3: print( - "Usage: python script.py " - " " + "Usage: python script.py " + " " ) sys.exit(1)