refact: support secrets from env vars and include vault namespace header

.github/workflows/build-pr-images.yml

@@ -27,15 +27,18 @@ jobs:
         id: pr
         run: echo "PR_NUMBER=${{ github.event.pull_request.number }}" >> $GITHUB_ENV
 
+      - name: Prepare image name
+        run: |
+          IMAGE_NAME="ghcr.io/${GITHUB_REPOSITORY,,}/authz:pr-${{ github.event.pull_request.number }}"
+          echo "IMAGE_NAME=$IMAGE_NAME" >> $GITHUB_ENV
+
       - name: Build Docker image
         run: |
-          IMAGE_NAME=ghcr.io/${{ github.repository }}/authz:pr-${PR_NUMBER}
           echo "Building image: $IMAGE_NAME"
           docker build -t $IMAGE_NAME .
 
       - name: Push Docker image
         run: |
-          IMAGE_NAME=ghcr.io/${{ github.repository }}/authz:pr-${PR_NUMBER}
           echo "Pushing image: $IMAGE_NAME"
           docker push $IMAGE_NAME
 
@@ -48,8 +51,8 @@ jobs:
     steps:
       - name: Delete Docker image from GHCR
         run: |
-          IMAGE_NAME=ghcr.io/${{ github.repository }}/authz:pr-${{ github.event.pull_request.number }}
+          IMAGE_NAME="ghcr.io/${GITHUB_REPOSITORY,,}/authz:pr-${{ github.event.pull_request.number }}"
           echo "Deleting image: $IMAGE_NAME"
           curl -X DELETE \
             -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" \
-            "https://ghcr.io/v2/${{ github.repository }}/authz/manifests/pr-${{ github.event.pull_request.number }}"
+            "https://ghcr.io/${GITHUB_REPOSITORY,,}/authz/manifests/pr-${{ github.event.pull_request.number }}"

Dockerfile

@@ -25,11 +25,14 @@ WORKDIR /vault/
 RUN mkdir -p /vault/config
 RUN mkdir -p /vault/data
 RUN chmod 777 /vault/data
+
+WORKDIR /app/
 RUN chown -R pcgl:pcgl /app
 
-USER pcgl
+RUN mkdir -p /permissions_engine
+RUN chown -R pcgl:pcgl /permissions_engine
 
-WORKDIR /app/
+USER pcgl
 
 RUN curl -L -o opa https://openpolicyagent.org/downloads/v1.1.0/opa_linux_amd64_static
 

README.md

@@ -32,7 +32,7 @@ curl -X "POST" "https://cilogon.org/oauth2/token" \
 
 3. Use the access token in an Authorization header for any of the API calls:
 ```
-curl "http://localhost:1235/authz/group/admin" \
+curl "http://localhost:1235/group/admin" \
  -H 'Authorization: Bearer <token>'
 ```
 

app/dev.entrypoint.sh

@@ -0,0 +1,70 @@
+#!/usr/bin/env bash
+
+# Container entrypoint for local development with root access to a local Vault server.
+
+set -Euo pipefail
+
+
+if [[ -f "/app/initial_setup" ]]; then
+
+    cp -r /app/permissions_engine/* /permissions_engine/
+    chmod 777 /permissions_engine
+
+    mkdir /app/data
+    chmod 777 /app/data
+
+    # set up our default values
+    sed -i s@PCGL_ADMIN_GROUP@$PCGL_ADMIN_GROUP@ /permissions_engine/calculate.rego
+
+    token=$(dd if=/dev/urandom bs=1 count=16 2>/dev/null | base64 | tr -d '\n\r+' | sed s/[^A-Za-z0-9]//g)
+    echo { \"opa_secret\": \"$token\" } > /permissions_engine/opa_secret.json
+    # set up vault URL and namespace
+    sed -i s@VAULT_URL@$VAULT_URL@ /permissions_engine/vault.rego
+    sed -i s@VAULT_NAMESPACE@$VAULT_NAMESPACE@ /permissions_engine/vault.rego
+
+    echo "initializing stores"
+    python3 /app/initialize_vault_store.py
+    if [[ $? -eq 0 ]]; then
+        rm /app/initial_setup
+        echo "setup complete"
+    else
+        echo "!!!!!! INITIALIZATION FAILED, TRY AGAIN !!!!!!"
+    fi
+
+else
+    sleep 10
+    # unseal vault
+    KEY=$(head -n 2 /app/config/keys.txt | tail -n 1)
+    echo '{ "key": "'$KEY'" }' > payload.json
+    curl --request POST --data @payload.json http://vault:8200/v1/sys/unseal
+    KEY=$(head -n 3 /app/config/keys.txt | tail -n 1)
+    echo '{ "key": "'$KEY'" }' > payload.json
+    curl --request POST --data @payload.json http://vault:8200/v1/sys/unseal
+    KEY=$(head -n 4 /app/config/keys.txt | tail -n 1)
+    echo '{ "key": "'$KEY'" }' > payload.json
+    curl --request POST --data @payload.json http://vault:8200/v1/sys/unseal
+fi
+
+# make sure that our vault stores have the latest values
+python3 /app/refresh_stores.py
+
+
+# start server
+cd /app/src
+gunicorn -k uvicorn.workers.UvicornWorker server:app &
+
+
+while [ 0 -eq 0 ]
+do
+  echo "storing vault token"
+  date
+  bash /app/renew_token.sh
+  python3 /app/refresh_stores.py
+  if [[ $? -eq 0 ]]; then
+      echo "vault token stored"
+      sleep 300
+  else
+      echo "vault token not stored"
+      sleep 30
+  fi
+done

app/entrypoint.sh

@@ -4,16 +4,21 @@ set -Euo pipefail
 
 
 if [[ -f "/app/initial_setup" ]]; then
+
+    cp -r /app/permissions_engine/* /permissions_engine/
+    chmod 777 /permissions_engine
+
     mkdir /app/data
     chmod 777 /app/data
 
     # set up our default values
-    sed -i s@PCGL_ADMIN_GROUP@$PCGL_ADMIN_GROUP@ /app/permissions_engine/calculate.rego
+    sed -i s@PCGL_ADMIN_GROUP@$PCGL_ADMIN_GROUP@ /permissions_engine/calculate.rego
 
     token=$(dd if=/dev/urandom bs=1 count=16 2>/dev/null | base64 | tr -d '\n\r+' | sed s/[^A-Za-z0-9]//g)
-    echo { \"opa_secret\": \"$token\" } > /app/permissions_engine/opa_secret.json
-    # set up vault URL
-    sed -i s@VAULT_URL@$VAULT_URL@ /app/permissions_engine/vault.rego
+    echo { \"opa_secret\": \"$token\" } > /permissions_engine/opa_secret.json
+    # set up vault URL and namespace
+    sed -i s@VAULT_URL@$VAULT_URL@ /permissions_engine/vault.rego
+    sed -i s@VAULT_NAMESPACE@$VAULT_NAMESPACE@ /permissions_engine/vault.rego
 
     echo "initializing stores"
     python3 /app/initialize_vault_store.py
@@ -23,18 +28,6 @@ if [[ -f "/app/initial_setup" ]]; then
     else
         echo "!!!!!! INITIALIZATION FAILED, TRY AGAIN !!!!!!"
     fi
-else
-    sleep 10
-    # unseal vault
-    KEY=$(head -n 2 /app/config/keys.txt | tail -n 1)
-    echo '{ "key": "'$KEY'" }' > payload.json
-    curl --request POST --data @payload.json http://vault:8200/v1/sys/unseal
-    KEY=$(head -n 3 /app/config/keys.txt | tail -n 1)
-    echo '{ "key": "'$KEY'" }' > payload.json
-    curl --request POST --data @payload.json http://vault:8200/v1/sys/unseal
-    KEY=$(head -n 4 /app/config/keys.txt | tail -n 1)
-    echo '{ "key": "'$KEY'" }' > payload.json
-    curl --request POST --data @payload.json http://vault:8200/v1/sys/unseal
 fi
 
 # make sure that our vault stores have the latest values

app/permissions_engine/vault.rego

@@ -17,20 +17,39 @@ test := "test" if {
 
 else := "opa"
 
+ns := "VAULT_NAMESPACE"
+
+vault_headers := {"X-Vault-Token": vault_token} if {
+    ns == ""
+}
+
+vault_headers := {"X-Vault-Token": vault_token, "X-Vault-Namespace": ns} if {
+    ns != ""
+}
+
+vault_service_headers := {"X-Vault-Token": input.token} if {
+    ns == ""
+}
+
+vault_service_headers := {"X-Vault-Token": input.token, "X-Vault-Namespace": ns} if {
+    ns != ""
+}
+
+
 # paths are the paths authorized for methods, used by permissions.rego
-paths := http.send({"method": "get", "url": concat("/", ["VAULT_URL/v1", test, "paths"]), "headers": {"X-Vault-Token": vault_token}}).body.data.paths
+paths := http.send({"method": "get", "url": concat("/", ["VAULT_URL/v1", test, "paths"]), "headers": vault_headers}).body.data.paths
 
 # groups are site-wide authorizations, used by permissions.rego and authz.rego
-groups := http.send({"method": "get", "url": concat("/", ["VAULT_URL/v1", test, "groups"]), "headers": {"X-Vault-Token": vault_token}}).body.data
+groups := http.send({"method": "get", "url": concat("/", ["VAULT_URL/v1", test, "groups"]), "headers": vault_headers}).body.data
 
-all_studies := http.send({"method": "get", "url": concat("/", ["VAULT_URL/v1", test, "studies"]), "headers": {"X-Vault-Token": vault_token}}).body.data.studies
+all_studies := http.send({"method": "get", "url": concat("/", ["VAULT_URL/v1", test, "studies"]), "headers": vault_headers}).body.data.studies
 
 study_auths[p] := study if {
 	some p in all_studies
-	study := http.send({"method": "get", "url": concat("/", ["VAULT_URL/v1", test, "studies", p]), "headers": {"X-Vault-Token": vault_token}}).body.data[p]
+	study := http.send({"method": "get", "url": concat("/", ["VAULT_URL/v1", test, "studies", p]), "headers": vault_headers}).body.data[p]
 }
 
-user_index := http.send({"method": "get", "url": concat("/", ["VAULT_URL/v1", test, "users/index"]), "headers": {"X-Vault-Token": vault_token}, "raise_error": false}).body.data
+user_index := http.send({"method": "get", "url": concat("/", ["VAULT_URL/v1", test, "users/index"]), "headers": vault_headers, "raise_error": false}).body.data
 
 user_id := user_index[data.idp.user_sub] if {
 	not input.body.user_pcglid
@@ -39,7 +58,7 @@ user_id := user_index[data.idp.user_sub] if {
 else := user_index[input.body.user_pcglid]
 
 # check to see if the user is authorized for any other studies via DACs
-user_auth := http.send({"method": "get", "url": concat("/", ["VAULT_URL/v1", test, "users", user_id]), "headers": {"X-Vault-Token": vault_token}, "raise_error": false})
+user_auth := http.send({"method": "get", "url": concat("/", ["VAULT_URL/v1", test, "users", user_id]), "headers": vault_headers, "raise_error": false})
 
 user_pcglid := user_auth.body.data.pcglid
 
@@ -51,4 +70,4 @@ user_studies := user_auth.body.data.study_authorizations if {
 
 default service := ""
 # if there is a service associated with this token:
-service := http.send({"method": "get", "url": concat("/", ["VAULT_URL/v1/cubbyhole", input.token]), "headers": {"X-Vault-Token": input.token}}).body.data.service
+service := http.send({"method": "get", "url": concat("/", ["VAULT_URL/v1/cubbyhole", input.token]), "headers": vault_service_headers}).body.data.service

app/refresh_stores.py

@@ -1,26 +1,29 @@
 import json
 import os
-from src.auth import get_vault_token_for_service, reload_comanage
+from src.auth import get_secret_file_value_or_env, get_vault_token_for_service, reload_comanage
 import sys
 import requests
 
-
 # get the token for the opa store
 try:
-    with open("/app/permissions_engine/opa_secret.json") as f:
+    with open("/permissions_engine/opa_secret.json") as f:
         opa_json = json.load(f)
         headers = {
             "X-Opa": opa_json["opa_secret"],
             "Content-Type": "application/json; charset=utf-8"
         }
-        with open("/home/pcgl/opa-roleid") as f:
-            role_id = f.read().strip()
-            payload = f"{{\"token\": \"{get_vault_token_for_service("opa", role_id=role_id)}\"}}"
-            response = requests.put(url=f"{os.getenv('OPA_URL')}/v1/data/opa_token", headers=headers, data=payload)
-            print(response.text)
-        with open("/home/pcgl/test-roleid") as f:
-            role_id = f.read().strip()
-            payload = f"{{\"token\": \"{get_vault_token_for_service("test", role_id=role_id)}\"}}"
+
+        # OPA Role-ID
+        opa_role_id = get_secret_file_value_or_env("/home/pcgl/opa-roleid", "OPA_ROLEID")
+        payload = f"{{\"token\": \"{get_vault_token_for_service("opa", role_id=opa_role_id)}\"}}"
+        response = requests.put(url=f"{os.getenv('OPA_URL')}/v1/data/opa_token", headers=headers, data=payload)
+        print(response.text)
+
+        # TODO: can we remove this? Feel that this should be in tests rather than baked in
+        # Test Role-ID
+        test_role_id = get_secret_file_value_or_env("/home/pcgl/test-roleid", "TEST_ROLEID")
+        if test_role_id:
+            payload = f"{{\"token\": \"{get_vault_token_for_service("test", role_id=test_role_id)}\"}}"
             response = requests.put(url=f"{os.getenv('OPA_URL')}/v1/data/test_token", headers=headers, data=payload)
             print(response.text)
 

app/renew_token.sh

@@ -1,18 +1,16 @@
 #!/usr/bin/env bash
 
 export VAULT_APPROLE_TOKEN=$(cat /vault/config/approle-token)
-export KEY_ROOT=$(tail -n 1 /vault/config/keys.txt)
-
 
 echo "renewing approle token"
-curl --request POST --header "X-Vault-Token: ${VAULT_APPROLE_TOKEN}" $VAULT_URL/v1/auth/token/renew-self > finish.json
+curl --request POST \
+    --header "X-Vault-Token: ${VAULT_APPROLE_TOKEN}" \
+    --header "X-Vault-Namespace: ${VAULT_NAMESPACE}" \
+    $VAULT_URL/v1/auth/token/renew-self > finish.json
 cat finish.json | jq
 grep "error" finish.json
 if [[ $? -eq 0 ]]; then
-    echo "creating approle token"
-    date
-    echo "{\"id\": \"${VAULT_APPROLE_TOKEN}\", \"policies\": [\"approle\"], \"periodic\": \"24h\"}" > token.json
-    curl --request POST --header "X-Vault-Token: ${KEY_ROOT}" --data @token.json $VAULT_URL/v1/auth/token/create/approle > finish.json
+    echo "Approle token renewal error:"
     cat finish.json | jq
 fi
 rm finish.json