Merge pull request #16 from artefactory/feat/remove_config

Feat/remove config

Author: Cedric-Magnan

.env.template

@@ -3,7 +3,6 @@ GH_APP_KEY="-----BEGIN RSA PRIVATE KEY-----
 Private Key data...
 -----END PRIVATE KEY-----"
 GH_PAT=""
-GDRIVE_MAIN_DIRECTORY_NAME=""
-USER_SHARE=""
+SQLALCHEMY_URI="sqlite:///database.db"
 ENV=""  # "LOCAL" or "GCP"
 GH_TESTS_REPO_NAME=""

.gitignore

@@ -21,6 +21,7 @@ Icon
 .terraform/
 terraform.tfstate
 terraform.tfstate.backup
+terraform.tfstate.*.backup
 .terraform.lock.hcl
 
 # Files that might appear in the root of a volume

README.md

@@ -65,16 +65,14 @@ With Cloud Run, you have an example terraform configuration [here](https://githu
 But you can deploy the application on many Serverless Container services on any cloud by making sure that :
 - The secrets defined in the `.env` file are available for the container at runtime as environment variables
 - The container can receive HTTP requests
-- The container can use a GCP service account to login with the [Python Google Auth client](https://google-auth.readthedocs.io/en/master/)
-- The service account is linked to a GCP Project which has the Google Drive API enabled
+- The container can login to any data warehouse with a SQLAlchemy Connection URI : [Bigquery](https://googleapis.dev/python/sqlalchemy-bigquery/latest/README.html#usage), [Snowflake](https://docs.snowflake.com/en/user-guide/sqlalchemy.html#connection-parameters), [Redshift](https://aws.amazon.com/fr/blogs/big-data/use-the-amazon-redshift-sqlalchemy-dialect-to-interact-with-amazon-redshift/) 
 
 ## Environment variables details
 
 - GH_APP_ID : Auto-generated ID of the GitHub App you created during the [`Prerequisites`](#prerequisites) step.
 - GH_APP_KEY : Private Key of the GitHub App you created during the [`Prerequisites`](#prerequisites) step.
 - GH_PAT : GitHub personal access token [you must create](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/creating-a-personal-access-token) that has access to the GitHub repository containing the tests and the original repository which was forked (both could be the same repository).
-- GDRIVE_MAIN_DIRECTORY_NAME : Name of the Google Drive Folder where you want the stats to be sent.
-- USER_SHARE : Comma-separated list of emails that have access to this Google Drive Folder.
+- SQLALCHEMY_URI : Database URI with [SQLAlchemy format](https://docs.sqlalchemy.org/en/14/core/engines.html#database-urls)
 - LOGGING : "LOCAL" if you are deploying locally, "GCP" if you are deploying on Google Cloud Run.
 - GH_TESTS_REPO_NAME : (Optional, only if you are using a git submodule for the tests folder) Name of the repository containing the tests (could be convenient if you have a repository with the exercices, and another one with the solutions and you want to have the same tests in both repositories by providing a submodule defined in a third repository).
 

docs/architecture.png

@@ -2,17 +2,25 @@
 
 echo "Please specify GCP project ID : "
 read PROJECT_ID
+echo "Please specify GCP region : "
+read REGION
 source .env
 gcloud config set project $PROJECT_ID
 gcloud auth application-default login
-export TF_project_id=$PROJECT_ID
+export TF_VAR_project_id=$PROJECT_ID
+export TF_VAR_region=$REGION
+export TF_VAR_docker_image="${REGION}-docker.pkg.dev/${PROJECT_ID}/github-app-registry/no_image"
 terraform -chdir=examples/cloud_run apply -input=true
 set +o history
 echo "$GH_APP_ID" | gcloud secrets versions add GH_APP_ID --data-file=-
 echo "$GH_APP_KEY" | gcloud secrets versions add GH_APP_KEY --data-file=-
 echo "$GH_PAT" | gcloud secrets versions add GH_PAT --data-file=-
 echo "$GH_TESTS_REPO_NAME" | gcloud secrets versions add GH_TESTS_REPO_NAME --data-file=-
-echo "$GDRIVE_MAIN_DIRECTORY_NAME" | gcloud secrets versions add GDRIVE_MAIN_DIRECTORY_NAME --data-file=-
-echo "$USER_SHARE" | gcloud secrets versions add USER_SHARE --data-file=-
+echo "$SQLALCHEMY_URI" | gcloud secrets versions add SQLALCHEMY_URI --data-file=-
 echo "$LOGGING" | gcloud secrets versions add LOGGING --data-file=-
 set -o history
+gcloud auth print-access-token | docker login -u oauth2accesstoken --password-stdin https://${REGION}-docker.pkg.dev
+docker build -t ${REGION}-docker.pkg.dev/${PROJECT_ID}/github-app-registry/github_tests_validator_app -f ./docker/Dockerfile .
+docker push ${REGION}-docker.pkg.dev/${PROJECT_ID}/github-app-registry/github_tests_validator_app
+export TF_VAR_docker_image=$(gcloud artifacts docker images list ${REGION}-docker.pkg.dev/${PROJECT_ID}/github-app-registry --filter="package=${REGION}-docker.pkg.dev/${PROJECT_ID}/github-app-registry/github_tests_validator_app" --sort-by="~UPDATE_TIME" --limit=1 --format="value(format("{0}@{1}",package,version))")
+terraform -chdir=examples/cloud_run apply -input=true

examples/cloud_run/deploy.sh

@@ -8,6 +8,11 @@ variable "region" {
     description = "GCP region where resources will be deployed"
 }
 
+variable "docker_image" {
+    type        = string
+    description = "Docker reference of the image used by Cloud Run"
+}
+
 terraform {
   required_providers {
     google = {
@@ -22,12 +27,6 @@ provider "google" {
   region      = "${var.region}"
 }
 
-resource "google_project_service" "drive_api_service" {
-  project = "${var.project_id}"
-  service = "drive.googleapis.com"
-  disable_dependent_services = true
-}
-
 resource "google_service_account" "service_account" {
   project = "${var.project_id}"
   account_id   = "github-tests-validator-app"
@@ -61,6 +60,24 @@ resource "google_project_iam_binding" "secret_accessor" {
   ]
 }
 
+resource "google_project_iam_binding" "bigquery_job_user" {
+  project = "${var.project_id}"
+  role    = "roles/bigquery.jobUser"
+
+  members = [
+    "serviceAccount:github-tests-validator-app@${var.project_id}.iam.gserviceaccount.com",
+  ]
+}
+
+resource "google_project_iam_binding" "bigquery_data_editor" {
+  project = "${var.project_id}"
+  role    = "roles/bigquery.dataEditor"
+
+  members = [
+    "serviceAccount:github-tests-validator-app@${var.project_id}.iam.gserviceaccount.com",
+  ]
+}
+
 resource "google_artifact_registry_repository" "github_test_validator_app_registry" {
   location      = "${var.region}"
   repository_id = "github-app-registry"
@@ -76,7 +93,7 @@ resource "google_cloud_run_service" "github_test_validator_app" {
             timeout_seconds = 300
             service_account_name = "github-tests-validator-app@${var.project_id}.iam.gserviceaccount.com"
             containers {
-                image = "${var.region}-docker.pkg.dev/${var.project_id}/github-app-registry/github_tests_validator_app:latest"
+                image = "${var.docker_image}"
                 env {
                     name = "GH_APP_ID"
                     value_from {
@@ -114,19 +131,10 @@ resource "google_cloud_run_service" "github_test_validator_app" {
                     }
                 }
                 env {
-                    name = "GDRIVE_MAIN_DIRECTORY_NAME"
-                    value_from {
-                        secret_key_ref {
-                            name = "GDRIVE_MAIN_DIRECTORY_NAME"
-                            key = "latest"
-                        }
-                    }
-                }
-                env {
-                    name = "USER_SHARE"
+                    name = "SQLALCHEMY_URI"
                     value_from {
                         secret_key_ref {
-                            name = "USER_SHARE"
+                            name = "SQLALCHEMY_URI"
                             key = "latest"
                         }
                     }
@@ -210,19 +218,8 @@ resource "google_secret_manager_secret" "GH_TESTS_REPO_NAME" {
     }
   }
 }
-resource "google_secret_manager_secret" "GDRIVE_MAIN_DIRECTORY_NAME" {
-  secret_id = "GDRIVE_MAIN_DIRECTORY_NAME"
-
-  replication {
-    user_managed {
-      replicas {
-        location = "${var.region}"
-      }
-    }
-  }
-}
-resource "google_secret_manager_secret" "USER_SHARE" {
-  secret_id = "USER_SHARE"
+resource "google_secret_manager_secret" "SQLALCHEMY_URI" {
+  secret_id = "SQLALCHEMY_URI"
 
   replication {
     user_managed {

examples/cloud_run/main.tf

@@ -4,28 +4,19 @@
 
 from github_tests_validator_app.bin.github_repo_validation import (
     get_event,
-    get_student_github_connector,
+    get_user_github_connector,
     validate_github_repo,
 )
-from github_tests_validator_app.bin.student_challenge_results_validation import (
-    send_student_challenge_results,
+from github_tests_validator_app.bin.user_pytest_summaries_validation import (
+    send_user_pytest_summaries,
 )
-from github_tests_validator_app.config import (
-    GDRIVE_MAIN_DIRECTORY_NAME,
-    GDRIVE_SUMMARY_SPREADSHEET,
-    GSHEET_DETAILS_SPREADSHEET,
-    USER_SHARE,
-)
-from github_tests_validator_app.lib.connectors.google_drive import GoogleDriveConnector
-from github_tests_validator_app.lib.connectors.google_sheet import GSheetConnector
-from github_tests_validator_app.lib.models.file import GSheetDetailFile, GSheetFile, WorkSheetFile
-from github_tests_validator_app.lib.models.users import GitHubUser
+from github_tests_validator_app.lib.connectors.sqlalchemy_client import SQLAlchemyConnector, User
 from github_tests_validator_app.lib.utils import init_github_user_from_github_event
 
 process = {
     "pull_request": validate_github_repo,
     "pusher": validate_github_repo,
-    "workflow_job": send_student_challenge_results,
+    "workflow_job": send_user_pytest_summaries,
 }
 
 
@@ -35,51 +26,12 @@ def handle_process(payload: Dict[str, Any]) -> str:
     if (
         not event
         or (event == "pull_request" and payload["action"] not in ["reopened", "opened"])
-        or (
-            event == "workflow_job"
-            and (
-                payload["action"] not in ["completed"]
-                or payload["workflow_job"]["conclusion"] != "success"
-            )
-        )
+        or (event == "workflow_job" and payload["action"] not in ["completed"])
     ):
         return ""
     return event
 
 
-def init_gsheet_file(
-    google_drive: GoogleDriveConnector,
-    info: Dict[str, Any],
-    parent_id: str,
-    shared_user_list: List[str],
-) -> GSheetFile:
-
-    gsheet = google_drive.get_gsheet(info["name"], parent_id, shared_user_list)
-    list_worksheets = [
-        WorkSheetFile(NAME=worksheet["name"], HEADERS=worksheet["headers"])
-        for _, worksheet in info["worksheets"].items()
-    ]
-    return GSheetFile(
-        NAME=info["name"],
-        MIMETYPE=gsheet.get("mimeType", ""),
-        ID=gsheet.get("id", ""),
-        WORKSHEETS=list_worksheets,
-    )
-
-
-def init_gsheet_detail_file(
-    google_drive: GoogleDriveConnector, info: Dict[str, Any], parent_id: str, user_share: List[str]
-) -> GSheetDetailFile:
-
-    gsheet = google_drive.get_gsheet(info["name"], parent_id, user_share)
-    return GSheetDetailFile(
-        NAME=info["name"],
-        MIMETYPE=gsheet.get("mimeType", ""),
-        ID=gsheet.get("id", ""),
-        HEADERS=info["headers"],
-    )
-
-
 def run(payload: Dict[str, Any]) -> None:
     """
     Validator function
@@ -95,42 +47,30 @@ def run(payload: Dict[str, Any]) -> None:
     if not event:
         return
 
-    # Init Google Drive connector and folders
-    google_drive = GoogleDriveConnector()
-    folder = google_drive.get_gdrive_folder(GDRIVE_MAIN_DIRECTORY_NAME, USER_SHARE)
-
-    # Init Google sheets
-    gsheet_summary_file = init_gsheet_file(
-        google_drive, GDRIVE_SUMMARY_SPREADSHEET, folder["id"], USER_SHARE
-    )
-    gsheet_details_file = init_gsheet_detail_file(
-        google_drive, GSHEET_DETAILS_SPREADSHEET, folder["id"], USER_SHARE
-    )
-
-    # Init Google sheet connector and worksheets
-    gsheet = GSheetConnector(google_drive.credentials, gsheet_summary_file, gsheet_details_file)
-
-    # Init GitHubUser
-    student_user = init_github_user_from_github_event(payload)
-    if not isinstance(student_user, GitHubUser):
+    # Init User
+    user_data = init_github_user_from_github_event(payload)
+    if not isinstance(user_data, dict):
         # Logging
         return
 
-    # Send user on Google Sheet
-    gsheet.add_new_user_on_sheet(student_user)
+    sql_client = SQLAlchemyConnector()
+
+    sql_client.add_new_user(user_data)
 
     # Check valid repo
-    student_github_connector = get_student_github_connector(student_user, payload)
-    if not student_github_connector:
-        gsheet.add_new_repo_valid_result(
-            student_user,
+    user_github_connector = get_user_github_connector(user_data, payload)
+    if not user_github_connector:
+        sql_client.add_new_repository_validation(
+            user_data,
             False,
-            "[ERROR]: cannot get the student github repository.",
+            payload,
+            event,
+            "[ERROR]: cannot get the user github repository.",
         )
-        logging.error("[ERROR]: cannot get the student github repository.")
+        logging.error("[ERROR]: cannot get the user github repository.")
         return
 
     logging.info(f'Begin process: "{event}"...')
     # Run the process
-    process[event](student_github_connector, gsheet, payload)
+    process[event](user_github_connector, sql_client, payload, event)
     logging.info(f'End of process: "{event}".')