Adding package structure for download tracker

lint

Beginning to add code to update index

Adding workflow and updating script to update index.html

Commenting out branch for now and renaming workflow

Adding id-token

Only adding new files to index

Adding dryrun and ability to create index file for first time

Fixing format of index.html

adding new line

using actual s3 link

Author: amontanez24

.github/workflows/publish_download_tracker.yml

@@ -0,0 +1,36 @@
+name: Publish Download Tracker
+on:
+  release:
+    types: [published]
+  pull_request:
+  workflow_dispatch:
+
+jobs:
+  publish-download-tracker:
+    runs-on: ubuntu-latest
+    permissions:
+      id-token: write
+    steps:
+      - uses: actions/checkout@v4
+        # with:
+        #   ref: 'stable'
+      - name: Set up latest Python
+        uses: actions/setup-python@v5
+        with:
+          python-version-file: 'pyproject.toml'
+      - name: Install dependencies
+        run: |
+            python -m pip install --upgrade pip
+            python -m pip install .[dev]
+            python -m pip install -r ./scripts/rdt-download-tracker/requirements.txt
+      - name: Configure AWS Credentials
+        uses: aws-actions/[email protected]
+        with:
+          aws-region: us-east-1
+          role-to-assume: ${{ secrets.AWS_DOWNLOAD_TRACKER_ROLE }}
+          role-session-name: UploadRDTDownloadTracker
+      - name: Create and upload
+        env:
+          DOWNLOAD_TRACKER_BUCKET: ${{ secrets.DOWNLOAD_TRACKER_BUCKET }}
+        run: |
+          python -m scripts.create_download_tracker

scripts/create_download_tracker.py

@@ -0,0 +1,102 @@
+"""Script that creates the rdt-download-tracker package."""
+
+import argparse
+import logging
+import os
+
+import boto3
+import build
+import tomli
+import tomli_w
+from botocore.exceptions import ClientError
+
+import rdt
+
+LOGGER = logging.getLogger(__name__)
+PROJECT_PATH = './scripts/rdt-download-tracker'
+BUCKET = os.getenv('DOWNLOAD_TRACKER_BUCKET', '')
+S3_PACKAGE_PATH = 'simple/rdt-download-tracker/'
+INDEX_FILE_NAME = 'index.html'
+
+
+def _set_version(version):
+    toml_path = os.path.join(PROJECT_PATH, 'pyproject.toml')
+    with open(toml_path, 'rb') as f:
+        pyproject = tomli.load(f)
+        pyproject['project']['version'] = version
+
+    with open(toml_path, 'wb') as f:
+        tomli_w.dump(pyproject, f)
+
+
+def build_package():
+    """Builds the wheel and sdist for 'rdt-download-tracker'."""
+    _set_version(rdt.__version__)
+    build.ProjectBuilder(PROJECT_PATH).build('wheel', 'dist')
+    build.ProjectBuilder(PROJECT_PATH).build('sdist', 'dist')
+
+
+def _load_local_index_file():
+    local_index_file_path = os.path.join(PROJECT_PATH, INDEX_FILE_NAME)
+    with open(local_index_file_path, 'rb') as local_index_file:
+        file = local_index_file.read().decode('utf-8')
+    return file
+
+
+def _update_index_html(files, s3_client, dryrun=False):
+    index_file_path = os.path.join(S3_PACKAGE_PATH, INDEX_FILE_NAME)
+    if not dryrun:
+        try:
+            response = s3_client.get_object(Bucket=BUCKET, Key=index_file_path)
+            current_index_file = response.get('Body').read().decode('utf-8')
+        except ClientError as e:
+            if e.response['Error']['Code'] == 'NoSuchKey':
+                LOGGER.info('Index file does not exist yet. Using local one instead.')
+                current_index_file = _load_local_index_file()
+    else:
+        current_index_file = _load_local_index_file()
+
+    insertion_point = current_index_file.find('</body>')
+    current_text = current_index_file[:insertion_point]
+    text_list = [current_text]
+    for file in files:
+        download_link = f'https://{BUCKET}.s3.us-east-1.amazonaws.com/{S3_PACKAGE_PATH}{file}'
+        new_link = f"<a href='{download_link}'>{file}</a>"
+        if new_link not in current_text:
+            text_list.append(new_link)
+            text_list.append('<br>')
+
+    text_list.append(current_index_file[insertion_point:])
+    new_index = '\n'.join(text_list)
+    if dryrun:
+        print('New index file:')  # noqa: T201 `print` found
+        print(new_index)  # noqa: T201 `print` found
+    else:
+        s3_client.put_object(Bucket=BUCKET, Key=index_file_path, Body=new_index)
+
+
+def upload_package(dryrun=False):
+    """Uploads the built package to the S3 bucket.
+
+    Args:
+        dryrun (bool):
+            If true, skip the actual uploading and just print out which files would be uploaded.
+    """
+    s3_client = boto3.client('s3')
+    files = os.listdir('dist')
+    for file_name in files:
+        dest = os.path.join(S3_PACKAGE_PATH, file_name)
+        if dryrun:
+            print(f'Uploading {file_name} as {dest} to bucket {BUCKET}')  # noqa: T201 `print` found
+        else:
+            s3_client.upload_file(os.path.join('dist', file_name), BUCKET, dest)
+
+    _update_index_html(files, s3_client, dryrun)
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument('-d', '--dryrun', action='store_true', help='Skip uploading built files.')
+    args = parser.parse_args()
+    build_package()
+    upload_package(args.dryrun)

scripts/rdt-download-tracker/README.md

@@ -0,0 +1 @@
+# TODO: fill this in

scripts/rdt-download-tracker/index.html

@@ -0,0 +1,8 @@
+<html>
+<head>
+    <title>Links</title>
+</head>
+<body>
+    <h1>Links</h1>
+</body>
+</html>

scripts/rdt-download-tracker/pyproject.toml

@@ -0,0 +1,12 @@
+[project]
+name = "rdt-download-tracker"
+description = "Package to track downloads of RDT"
+version = "1.18.1.dev0"
+readme = "README.md"
+
+[build-system]
+requires = [
+    "setuptools",
+    "wheel",
+]
+build-backend = "setuptools.build_meta"

scripts/rdt-download-tracker/rdt_download_tracker/__init__.py

@@ -0,0 +1 @@
+"""Package for tracking downloads of RDT."""

scripts/rdt-download-tracker/requirements.txt

@@ -0,0 +1,4 @@
+boto3
+build
+tomli
+tomli-w