Merge branch 'master' into k8s-helm

c-simpson · c-simpson · commit 238da664183d · 2021-04-26T21:18:33.000-04:00
diff --git a/.github/workflows/release-containers.yml b/.github/workflows/release-containers.yml
@@ -0,0 +1,63 @@
+name: Release Containers
+
+on:
+  push:
+    tags: [ 'v*' ]
+
+
+jobs:
+  release-containers:
+    name: Build and Push
+    runs-on: ubuntu-latest
+    steps:
+
+    - uses: actions/checkout@v2
+
+    - name: Login to ghcr.io Docker registry
+      uses: docker/login-action@v1
+      with:
+        registry: ghcr.io
+        username: ${{ github.repository_owner }}
+        password: ${{ secrets.GITHUB_TOKEN }}
+
+    - name: Compute Docker container image addresses
+      run: |
+        DOCKER_REPOSITORY="ghcr.io/${GITHUB_REPOSITORY,,}"
+        DOCKER_TAG="${GITHUB_REF:11}"
+        echo "DOCKER_REPOSITORY=${DOCKER_REPOSITORY}" >> $GITHUB_ENV
+        echo "DOCKER_TAG=${DOCKER_TAG}" >> $GITHUB_ENV
+        echo "Using: ${DOCKER_REPOSITORY}/*:${DOCKER_TAG}"
+
+    - name: 'Pull latest existing Docker container image: server'
+      run: docker pull "${DOCKER_REPOSITORY}/server:latest" || true
+
+    - name: 'Pull latest existing Docker container image: client'
+      run: docker pull "${DOCKER_REPOSITORY}/client:latest" || true
+
+    - name: 'Build Docker container image: server'
+      run: |
+        docker build \
+            --cache-from "${DOCKER_REPOSITORY}/server:latest" \
+            --tag "${DOCKER_REPOSITORY}/server:latest" \
+            --tag "${DOCKER_REPOSITORY}/server:${DOCKER_TAG}" \
+            ./src/server
+
+    - name: Push Docker container image server:latest"
+      run: docker push "${DOCKER_REPOSITORY}/server:latest"
+
+    - name: Push Docker container image server:v*"
+      run: docker push "${DOCKER_REPOSITORY}/server:${DOCKER_TAG}"
+
+    - name: 'Build Docker container image: client'
+      run: |
+        docker build \
+            --cache-from "${DOCKER_REPOSITORY}/client:latest" \
+            --tag "${DOCKER_REPOSITORY}/client:latest" \
+            --tag "${DOCKER_REPOSITORY}/client:${DOCKER_TAG}" \
+            ./src/client
+
+    - name: Push Docker container image client:latest"
+      run: docker push "${DOCKER_REPOSITORY}/client:latest"
+
+    - name: Push Docker container image client:v*"
+      run: docker push "${DOCKER_REPOSITORY}/client:${DOCKER_TAG}"
diff --git a/src/client/src/components/Login/Login.js b/src/client/src/components/Login/Login.js
@@ -12,7 +12,7 @@ function checkLoginResponse(response) {
 
 
 async function loginUser(credentials) {
-    return fetch('http://localhost:5000/api/user/login', {
+    return fetch('/api/user/login', {
         method: 'POST',
         headers: {
             'Content-Type': 'application/json'
diff --git a/src/client/src/components/Refresh.js b/src/client/src/components/Refresh.js
@@ -2,7 +2,7 @@ export default async function Refresh(old_token) {
     
     // Use exisiting token to get a new fresh token
 
-        const new_at = await fetch('http://localhost:5000/api/user/refresh',
+        const new_at = await fetch('/api/user/refresh',
         {
             method: 'GET',
             headers: {
diff --git a/src/client/src/pages/Admin.js b/src/client/src/pages/Admin.js
@@ -55,7 +55,7 @@ class Admin extends Component {
         await this.handleGetFileList();
         await this.handleGetStatistics();
 
-        this.setState({isLoading: false})
+        await this.setState({isLoading: false})
     }
 
     componentDidMount() {
@@ -78,14 +78,13 @@ class Admin extends Component {
             formData.append('file', element, element.name)
         })
 
-        await fetch("/api/file", {method: 'POST', body: formData, 
+        await fetch("/api/file", {
+            method: 'POST',
+            body: formData,
             headers: {
-                'Content-Type': 'application/json',
                 'Authorization': 'Bearer ' + this.props.access_token
             }})
 
-
-
         await this.handleGetFileList();
 
         await this.setState({
@@ -105,7 +104,6 @@ class Admin extends Component {
             {
                 method: 'GET',
                 headers: {
-                    'Content-Type': 'application/json',
                     'Authorization': 'Bearer ' + this.props.access_token
                 }
             });
@@ -118,7 +116,6 @@ class Admin extends Component {
             {
                 method: 'GET',
                 headers: {
-                    'Content-Type': 'application/json',
                     'Authorization': 'Bearer ' + this.props.access_token
                 }
             });
@@ -139,7 +136,6 @@ class Admin extends Component {
             {
                 method: 'GET',
                 headers: {
-                    'Content-Type': 'application/json',
                     'Authorization': 'Bearer ' + this.props.access_token
                 }
             });
diff --git a/src/client/src/pages/Check/Check.js b/src/client/src/pages/Check/Check.js
@@ -35,7 +35,7 @@ export default function Check({access_token}) {
 
         function authCheck() {
             //   console.log('authCheck startfetch @ ' + DateTime.local().toFormat('HH:mm:ss.SSS'))
-               fetch('http://localhost:5000/api/user/test_auth',
+               fetch('/api/user/test_auth',
                {
                    method: 'GET',
                    headers: {
diff --git a/src/server/alembic.ini b/src/server/alembic.ini
@@ -36,7 +36,7 @@ script_location = alembic
 # output_encoding = utf-8
 
 # Container
-sqlalchemy.url = postgresql://postgres:thispasswordisverysecure@paws-compose-db/paws
+sqlalchemy.url = postgresql://postgres:PASSWORD@paws-compose-db/paws
 
 # Local
 # sqlalchemy.url = postgresql://postgres:thispasswordisverysecure@localhost/paws
diff --git a/src/server/alembic/env.py b/src/server/alembic/env.py
@@ -3,6 +3,8 @@
 from sqlalchemy import engine_from_config
 from sqlalchemy import pool
 
+from os import environ
+
 from alembic import context
 
 # this is the Alembic Config object, which provides
@@ -25,6 +27,13 @@
 # ... etc.
 
 
+PG_URL1 = 'postgresql://postgres:'
+PG_URL2 = environ['POSTGRES_PASSWORD']
+PG_URL3 = '@paws-compose-db/paws'
+
+PG_URL = PG_URL1 + PG_URL2 + PG_URL3
+
+
 def run_migrations_offline():
     """Run migrations in 'offline' mode.
 
@@ -37,7 +46,8 @@ def run_migrations_offline():
     script output.
 
     """
-    url = config.get_main_option("sqlalchemy.url")
+    # url = config.get_main_option("sqlalchemy.url")
+    url = PG_URL
     context.configure(
         url=url,
         target_metadata=target_metadata,
@@ -60,6 +70,7 @@ def run_migrations_online():
         config.get_section(config.config_ini_section),
         prefix="sqlalchemy.",
         poolclass=pool.NullPool,
+        url=PG_URL,
     )
 
     with connectable.connect() as connection:
diff --git a/src/server/api/admin_api.py b/src/server/api/admin_api.py
@@ -1,5 +1,4 @@
 from api.api import admin_api
-import shutil
 import os
 from datetime import datetime
 import json
@@ -16,23 +15,19 @@
 from config import (
     RAW_DATA_PATH,
     CURRENT_SOURCE_FILES_PATH,
-    LOGS_PATH,
 )
 
 ALLOWED_EXTENSIONS = {"csv", "xlsx"}
 
+
 def __allowed_file(filename):
     return "." in filename and filename.rsplit(".", 1)[1].lower() in ALLOWED_EXTENSIONS
 
 
-
 # file upload tutorial
 @admin_api.route("/api/file", methods=["POST"])
 @jwt_ops.admin_required
-def uploadCSV():
-    if "file" not in request.files:
-        return redirect(request.url)
-
+def upload_csv():
     for file in request.files.getlist("file"):
         if __allowed_file(file.filename):
             try:
diff --git a/src/server/api/common_api.py b/src/server/api/common_api.py
@@ -7,7 +7,23 @@
 import time
 from datetime import datetime
 import dateutil.parser
-from secrets import SHELTERLUV_SECRET_TOKEN
+
+
+try:   
+    from secrets import SHELTERLUV_SECRET_TOKEN
+except ImportError:   
+    # Not running locally
+    print("Couldn't get SL_TOKEN from file, trying environment **********")
+    from os import environ
+
+    try:
+        SHELTERLUV_SECRET_TOKEN = environ['SHELTERLUV_SECRET_TOKEN']
+    except KeyError:
+        # Nor in environment
+        # You're SOL for now
+        print("Couldn't get secrets from file or environment")
+
+
 
 from api import jwt_ops
 
diff --git a/src/server/app.py b/src/server/app.py
@@ -4,7 +4,21 @@
 
 from flask_jwt_extended import JWTManager
 
-from secrets import JWT_SECRET, APP_SECRET_KEY
+try:   
+    from secrets import JWT_SECRET, APP_SECRET_KEY
+except ImportError:   
+    # Not running locally
+    print("Could not get secrets from file, trying environment **********")
+    from os import environ
+
+    try:
+        JWT_SECRET = environ['JWT_SECRET']
+        APP_SECRET_KEY = environ['APP_SECRET_KEY']
+    except KeyError:
+        # Nor in environment
+        # You're SOL for now
+        print("Couldn't get secrets from file or environment")
+
 
 app = Flask(__name__)
 
diff --git a/src/server/bin/export_secrets.sh b/src/server/bin/export_secrets.sh
@@ -0,0 +1,7 @@
+set -o allexport
+ while read k _ v; 
+    do 
+      eval $k=$v;
+      export k;
+    done < 'secrets.py'
+ set +o allexport
diff --git a/src/server/bin/startServer.sh b/src/server/bin/startServer.sh
@@ -1,9 +1,10 @@
+#!/bin/bash
 
 # we may want to switch this to a script which logs output, etc?
 echo "------------STARTING `date` ------------------"
 set FLASK_APP=server/app.py
 export FLASK_APP
-
+source bin/export_secrets.sh
 # This abomination ensures that the PG server has finished its restart cycle
 echo "SLEEPING.. WAITING FOR DB"; sleep 5; echo "WAKING"; alembic upgrade head; echo "DB SETUP";
 #; python -m flask run --host=0.0.0.0 --no-reload
diff --git a/src/server/pipeline/shelterluv_api_handler.py b/src/server/pipeline/shelterluv_api_handler.py
@@ -1,7 +1,21 @@
 import requests
 import csv
 from config import RAW_DATA_PATH
-from secrets import SHELTERLUV_SECRET_TOKEN
+
+try:   
+    from secrets import SHELTERLUV_SECRET_TOKEN
+except ImportError:   
+    # Not running locally
+    print("Couldn't get SL_TOKEN from file, trying environment **********")
+    from os import environ
+
+    try:
+        SHELTERLUV_SECRET_TOKEN = environ['SHELTERLUV_SECRET_TOKEN']
+    except KeyError:
+        # Nor in environment
+        # You're SOL for now
+        print("Couldn't get secrets from file or environment")
+
 
 
 def write_csv(json_data):
diff --git a/src/server/requirements.txt b/src/server/requirements.txt
@@ -1,10 +1,8 @@
 Flask==1.1.2
 pandas==1.0.0
 numpy==1.18.1
-fuzzywuzzy==0.17.0
 sqlalchemy < 1.4.0   # Import error with 1.4.0 on _ColumnEntity
 psycopg2-binary==2.8.4
-python-Levenshtein-wheels
 xlrd==1.2.0  # currently used for xlsx, but we should consider adjusting code to openpyxl for xlsx
 openpyxl
 requests
diff --git a/src/server/user_mgmt/base_users.py b/src/server/user_mgmt/base_users.py
@@ -3,7 +3,25 @@
 from api import user_api
 import sqlalchemy as sa
 
-from secrets import BASEUSER_PW, BASEEDITOR_PW, BASEADMIN_PW
+try:   
+    from secrets import BASEUSER_PW, BASEEDITOR_PW, BASEADMIN_PW
+except ImportError:   
+    # Not running locally
+    print("Couldn't get BASE user PWs from file, trying environment **********")
+    from os import environ
+
+    try:
+        BASEUSER_PW = environ['BASEUSER_PW']
+        BASEEDITOR_PW = environ['BASEEDITOR_PW']
+        BASEADMIN_PW = environ['BASEADMIN_PW']
+
+    except KeyError:
+        # Nor in environment
+        # You're SOL for now
+        print("Couldn't get secrets from file or environment")
+
+
+
 
 
 from sqlalchemy import Table, Column, Integer, String, MetaData, ForeignKey

Original file line number	Diff line number	Diff line change
`@@ -2,7 +2,7 @@ export default async function Refresh(old_token) {`
`2`	`2`
`3`	`3`	`// Use exisiting token to get a new fresh token`
`4`	`4`
`5`		`- const new_at = await fetch('http://localhost:5000/api/user/refresh',`
	`5`	`+ const new_at = await fetch('/api/user/refresh',`
`6`	`6`	`{`
`7`	`7`	`method: 'GET',`
`8`	`8`	`headers: {`
Original file line number	Diff line number	Diff line change
`@@ -35,7 +35,7 @@ export default function Check({access_token}) {`
`35`	`35`
`36`	`36`	`function authCheck() {`
`37`	`37`	`// console.log('authCheck startfetch @ ' + DateTime.local().toFormat('HH:mm:ss.SSS'))`
`38`		`- fetch('http://localhost:5000/api/user/test_auth',`
	`38`	`+ fetch('/api/user/test_auth',`
`39`	`39`	`{`
`40`	`40`	`method: 'GET',`
`41`	`41`	`headers: {`