Merge branch 'chaoss:main' into d0-contributor-metric

Author: Akshatb2006

.github/workflows/build_docker.yml

@@ -52,7 +52,7 @@ jobs:
 
 
   test-e2e:
-    name: End-to-end test
+    name: End-to-end test (Docker)
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repository
@@ -168,6 +168,121 @@ jobs:
         # We use tail so that we can see the name of each file as it's printed
         run: "docker run -t --rm -v augur_logs:/logs bash -c 'find /logs -type f | xargs tail -n +0'"
 
+  test-e2e-podman:
+    name: End-to-end test (Podman)
+    runs-on: ubuntu-latest
+    steps:
+      - name: Remove unnecessary files from the base image
+        run: |
+          sudo rm -rf /usr/share/dotnet
+          sudo rm -rf "$AGENT_TOOLSDIRECTORY"
+
+      - name: Start Podman socket
+        run: systemctl --user start podman.socket
+
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Build database container
+        uses: redhat-actions/buildah-build@v2
+        with:
+          context: .
+          containerfiles: |
+            ./docker/database/Dockerfile
+          platforms: linux/amd64
+          tags: ghcr.io/${{ github.repository_owner }}/augur_database:test
+          layers: true
+
+      - name: Build keyman container
+        uses: redhat-actions/buildah-build@v2
+        with:
+          context: .
+          containerfiles: |
+            ./docker/keyman/Dockerfile
+          platforms: linux/amd64
+          tags: ghcr.io/${{ github.repository_owner }}/augur_keyman:test
+          layers: true
+
+      - name: Build rabbitmq container
+        uses: redhat-actions/buildah-build@v2
+        with:
+          context: .
+          containerfiles: |
+            ./docker/rabbitmq/Dockerfile
+          platforms: linux/amd64
+          tags: ghcr.io/${{ github.repository_owner }}/augur_rabbitmq:test
+          layers: true
+
+      - name: Build backend container
+        uses: redhat-actions/buildah-build@v2
+        with:
+          context: .
+          containerfiles: |
+            ./docker/backend/Dockerfile
+          platforms: linux/amd64
+          tags: ghcr.io/${{ github.repository_owner }}/augur_backend:test
+          layers: true
+
+      - name: Prepare compose file
+        run: |
+          yq eval -i '.services.augur.image = "ghcr.io/${{ github.repository_owner }}/augur_backend:test"' docker-compose.yml
+          yq eval -i '.services.augur.pull_policy = "never"' docker-compose.yml
+          yq eval -i '.services.augur.restart = "no"' docker-compose.yml
+
+          yq eval -i '.services.augur-db.image = "ghcr.io/${{ github.repository_owner }}/augur_database:test"' docker-compose.yml
+          yq eval -i '.services.augur-db.pull_policy = "never"' docker-compose.yml
+          yq eval -i '.services.augur-db.restart = "no"' docker-compose.yml
+
+          yq eval -i '.services.augur-keyman.image = "ghcr.io/${{ github.repository_owner }}/augur_keyman:test"' docker-compose.yml
+          yq eval -i '.services.augur-keyman.pull_policy = "never"' docker-compose.yml
+          yq eval -i '.services.augur-keyman.restart = "no"' docker-compose.yml
+
+          yq eval -i '.services.rabbitmq.image = "ghcr.io/${{ github.repository_owner }}/augur_rabbitmq:test"' docker-compose.yml
+          yq eval -i '.services.rabbitmq.pull_policy = "never"' docker-compose.yml
+          yq eval -i '.services.rabbitmq.restart = "no"' docker-compose.yml
+
+      - name: Setup Podman Compose
+        uses: webgtx/setup-podman-compose@v1
+
+      - name: Set up list of log lines to match
+        run: |
+          cat <<EOF > /tmp/regex_matches.txt
+          Gunicorn webserver started
+          Starting core worker processes
+          Starting secondary worker processes
+          Starting facade worker processes
+          Retrieved \\d+ github api keys for use
+          Fetching new repos \\(complete\\)
+          Inserting \\d+ contributors
+          Inserting \\d+ issues
+          Inserting prs of length: \\d+
+          Querying committers count
+          Done generating scc data for repo
+          Sending due task
+          EOF
+
+      - name: Start services & wait for output
+        # This starts the system and sends the output to "await_all.py" which
+        # scans for the regex matches from above. Once all matches are seen at
+        # least once, the `compose down` will run to shut down the system. If
+        # this all doesn't happen before the timeout, the job will fail.
+        run: |
+          podman compose -f docker-compose.yml up --no-build 2>&1 \
+            | (./scripts/ci/await_all.py /tmp/regex_matches.txt \
+                && docker compose -f docker-compose.yml down)
+        timeout-minutes: 3
+        env:
+          AUGUR_GITHUB_API_KEY: ${{ secrets.GITHUB_TOKEN }}
+          AUGUR_GITHUB_USERNAME: ${{ github.repository_owner }}
+          AUGUR_GITLAB_API_KEY: dummy
+          AUGUR_GITLAB_USERNAME: dummy
+
+      - name: Dump logs
+        # Always run this step to get logs, even if the previous step fails
+        if: always()
+        # We use tail so that we can see the name of each file as it's printed
+        run: "podman run -t --rm -v augur_logs:/logs bash -c 'find /logs -type f | xargs tail -n +0'"
+
 
 
   push-image:

.github/workflows/checks.yml

@@ -4,6 +4,17 @@ on:
     branches: [main, release]
 
 jobs:
+  check-docs:
+    name: runner / check docs
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Install uv
+        uses: astral-sh/setup-uv@v6
+      - name: Ensure docs build cleanly
+        # Setting `O` to pass extra options to the sphinx-build command.
+        run: O="-a -E -n -W --keep-going" make docs
+
   run-pylint:
     name: runner / pylint
     permissions: write-all

CONTRIBUTING.md

@@ -90,8 +90,8 @@ git push origin master
 ## Community Resources
 
 ### Augur
-- [Stable documentation (`main` branch)](https://oss-augur.readthedocs.io/en/main/)
-- [Nightly/developer build documentation (`dev` branch)](https://oss-augur.readthedocs.io/en/dev/) (warning: this is should be considered an unstable branch and should not be used for production)
+- [Stable documentation (`release` branch)](https://oss-augur.readthedocs.io/en/release/)
+- [Nightly/developer build documentation (`main` branch)](https://oss-augur.readthedocs.io/en/main/) (warning: this is should be considered an unstable branch and should not be used for production)
 - [Live Augur demo](https://ai.chaoss.io)
 
 ### CHAOSS

README.md

@@ -6,15 +6,15 @@ The primary way of looking at Augur data is through [8Knot](https://github.com/o
 [![first-timers-only](https://img.shields.io/badge/first--timers--only-friendly-blue.svg?style=flat-square)](https://www.firsttimersonly.com/)
 We follow the [First Timers Only](https://www.firsttimersonly.com/) philosophy of tagging issues for first timers only, and walking one newcomer through the resolution process weekly. You can find these issues tagged with [first timers only](https://github.com/chaoss/augur/labels/first-timers-only) on our issues list.
 
-[![standard-readme compliant](https://img.shields.io/badge/standard--readme-OK-green.svg?style=flat-square)](https://github.com/RichardLitt/standard-readme) [![Build Docker images](https://github.com/chaoss/augur/actions/workflows/build_docker.yml/badge.svg)](https://github.com/chaoss/augur/actions/workflows/build_docker.yml) [![Hits-of-Code](https://hitsofcode.com/github/chaoss/augur?branch=main)](https://hitsofcode.com/github/chaoss/augur/view?branch=main) [![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/2788/badge)](https://bestpractices.coreinfrastructure.org/projects/2788)
+[![standard-readme compliant](https://img.shields.io/badge/standard--readme-OK-green.svg?style=flat-square)](https://github.com/RichardLitt/standard-readme) [![Build Docker images](https://github.com/chaoss/augur/actions/workflows/build_docker.yml/badge.svg)](https://github.com/chaoss/augur/actions/workflows/build_docker.yml) [![Hits-of-Code](https://hitsofcode.com/github/chaoss/augur?branch=release)](https://hitsofcode.com/github/chaoss/augur/view?branch=release) [![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/2788/badge)](https://bestpractices.coreinfrastructure.org/projects/2788)
 
 ## NEW RELEASE ALERT!
 **If you want to jump right in, the updated docker, docker-compose and bare metal installation instructions are available [here](docs/new-install.md)**.
 
-Augur is now releasing a dramatically improved new version to the ```main``` branch. It is also available [here](https://github.com/chaoss/augur/releases/tag/v0.89.3).
+Augur is now releasing a dramatically improved new version. It is also available [here](https://github.com/chaoss/augur/releases/tag/v0.89.3).
 
 
-- The `main` branch is a stable version of our new architecture, which features:
+- The `release` branch is a stable version of our new architecture, which features:
   - Dramatic improvement in the speed of large scale data collection (100,000+ repos). All data is obtained for 100k+ repos within 2 weeks.
   - A new job management architecture that uses Celery and Redis to manage queues, and enables users to run a Flower job monitoring dashboard.
   - Materialized views to increase the snappiness of API’s and Frontends on large scale data.

augur/api/gunicorn_conf.py

@@ -1,6 +1,7 @@
 # from augur import ROOT_AUGUR_DIRECTORY
 import multiprocessing
 import logging
+import os
 from pathlib import Path
 from glob import glob
 
@@ -19,7 +20,22 @@
 workers = multiprocessing.cpu_count() * 2 + 1
 umask = 0o007
 reload = True
-reload_extra_files = glob(str(Path.cwd() / '**/*.j2'), recursive=True)
+
+is_dev = os.getenv("AUGUR_DEV", 'False').lower() in ('true', '1', 't', 'y', 'yes')
+
+if is_dev:
+
+    augur_templates_dir = Path.cwd() / "augur/templates"
+
+    if not augur_templates_dir.is_dir():
+        logger.critical("Could not locate templates in Gunicorn startup")
+        exit(-1)
+
+    reload_extra_files = glob(str(augur_templates_dir.resolve() / '**/*.j2'), recursive=True)
+
+    # Don't  want to leave extraneous variables in config scope
+    del augur_templates_dir
+del is_dev
 
 # set the log location for gunicorn    
 logs_directory = get_value('Logging', 'logs_directory')

augur/api/routes/user.py

@@ -16,7 +16,8 @@
 from augur.api.util import ssl_required
 
 from augur.application.db.models import User, UserSessionToken, RefreshToken
-from augur.tasks.init.redis_connection import redis_connection as redis
+from augur.tasks.init.redis_connection import get_redis_connection
+
 from ..server import app
 
 logger = logging.getLogger(__name__)
@@ -64,6 +65,8 @@ def user_authorize():
     code = secrets.token_hex()
     username = current_user.login_name
 
+    redis = get_redis_connection()
+
     redis.set(code, username, ex=300)
 
     return jsonify({"status": "Validated", "code": code})
@@ -81,6 +84,8 @@ def generate_session(application):
 
     if "code" not in grant_type:
         return jsonify({"status": "Invalid grant type"})
+    
+    redis = get_redis_connection()
 
     username = redis.get(code)
     redis.delete(code)

augur/application/cli/_cli_util.py

@@ -5,14 +5,17 @@
 import signal
 from urllib.parse import urlparse
 
-from augur.tasks.init.redis_connection import redis_connection 
+from augur.tasks.init.redis_connection import get_redis_connection
 
 def clear_redis_caches(logger):
     """Clears the redis databases that celery and redis use."""
 
     logger.info("Flushing all redis databases this instance was using")
     celery_purge_command = "celery -A augur.tasks.init.celery_app.celery_app purge -f"
     subprocess.call(celery_purge_command.split(" "))
+
+
+    redis_connection = get_redis_connection()
     redis_connection.flushdb()
 
 

augur/application/cli/backend.py

@@ -22,7 +22,7 @@
 from augur.tasks.github.util.github_api_key_handler import GithubApiKeyHandler
 from augur.tasks.gitlab.gitlab_api_key_handler import GitlabApiKeyHandler
 from augur.tasks.data_analysis.contributor_breadth_worker.contributor_breadth_worker import contributor_breadth_model
-from augur.tasks.init.redis_connection import redis_connection 
+from augur.tasks.init.redis_connection import get_redis_connection 
 from augur.application.db.models import UserRepo
 from augur.application.db.session import DatabaseSession
 from augur.application.logs import AugurLogger
@@ -32,7 +32,9 @@
 
 from keyman.KeyClient import KeyClient, KeyPublisher
 
-logger = AugurLogger("augur", reset_logfiles=True).get_logger()
+reset_logs = os.getenv("AUGUR_RESET_LOGS", 'True').lower() in ('true', '1', 't', 'y', 'yes')
+
+logger = AugurLogger("augur", reset_logfiles=reset_logs).get_logger()
 
 
 @click.group('server', short_help='Commands for controlling the backend API server & data collection workers')
@@ -363,6 +365,8 @@ def clear_redis_caches():
     logger.info("Flushing all redis databases this instance was using")
     celery_purge_command = "celery -A augur.tasks.init.celery_app.celery_app purge -f"
     subprocess.call(celery_purge_command.split(" "))
+
+    redis_connection = get_redis_connection()
     redis_connection.flushdb()
 
 def clear_all_message_queues(connection_string):

augur/application/cli/cache.py

@@ -4,7 +4,7 @@
 """
 import click
 
-from augur.tasks.init.redis_connection import redis_connection as redis_conn
+from augur.tasks.init.redis_connection import get_redis_connection
 from augur.application.logs import AugurLogger
 from augur.application.cli import test_connection, test_db_connection 
 
@@ -30,6 +30,7 @@ def clear_all():
 
         if user_input in ("y", "Y", "Yes", "yes"):
             logger.info("Clearing call redis databases")
+            redis_conn = get_redis_connection()
             redis_conn.flushall()
             return
 
@@ -46,4 +47,6 @@ def clear():
     """Clears the redis cache specified in the config"""
 
     print("Clearing redis cache that is specified in the config")
+
+    redis_conn = get_redis_connection()
     redis_conn.flushdb()

augur/application/cli/tasks.py

@@ -13,7 +13,6 @@
 
 from augur import instance_id
 from augur.application.logs import AugurLogger
-from augur.tasks.init.redis_connection import redis_connection
 from augur.application.cli import test_connection, test_db_connection 
 from augur.application.cli.backend import clear_rabbitmq_messages, raise_open_file_limit