Merge pull request #1474 from ImagingDataCommons/idc-prod-sp

Release 46 / v21

Author: s-paquette

.circleci/config.yml

@@ -119,7 +119,7 @@ jobs:
       TZ: "/usr/share/zoneinfo/America/Los_Angeles"
     working_directory: ~/IDC-WebApp
     docker:
-      - image: cimg/python:3.9.2
+      - image: cimg/python:3.11
       - image: cimg/mysql:8.0
         environment:
           MYSQL_ROOT_HOST: "%"
@@ -195,7 +195,7 @@ jobs:
       TZ: "/usr/share/zoneinfo/America/Los_Angeles"
     working_directory: ~/IDC-WebApp
     docker:
-      - image: cimg/python:3.9.2
+      - image: cimg/python:3.11
     steps:
       - restore_cache:
           keys:
@@ -209,7 +209,7 @@ jobs:
             sudo -E rm -rf ./lib
             sudo -E /bin/bash ./shell/unpack_for_deployment.sh
             sudo -E gcloud config set app/cloud_build_timeout 1600
-            sudo -E gcloud app deploy --verbosity=debug ./app.yaml --quiet
+            sudo -E gcloud app deploy --verbosity=debug ./app.yaml --service-account=${WEBAPP_RUNTIME_SA_NAME} --quiet
 workflows:
   version: 2
   build_and_deploy:

Dockerfile

@@ -1,6 +1,6 @@
 ###
 #
-# Copyright 2017, Institute for Systems Biology
+# Copyright 2025, Institute for Systems Biology
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -18,7 +18,7 @@
 
 # Dockerfile extending the Python Community image from Dockerhub with application files for a
 # single application.
-FROM python:3.9-bullseye
+FROM python:3.11-bookworm
 
 SHELL ["/bin/bash", "-c"]
 
@@ -29,12 +29,12 @@ RUN apt-get update
 RUN apt-get install -y wget
 # TODO: we need to start using the keyring instead
 RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv A8D3785C
-RUN wget "http://repo.mysql.com/mysql-apt-config_0.8.29-1_all.deb" -P /tmp
+RUN wget "http://repo.mysql.com/mysql-apt-config_0.8.30-1_all.deb" -P /tmp
 
 # install lsb-release (a dependency of mysql-apt-config), since dpkg doesn't
 # do dependency resolution
 RUN apt-get install -y lsb-release
-RUN dpkg --install /tmp/mysql-apt-config_0.8.29-1_all.deb
+RUN dpkg --install /tmp/mysql-apt-config_0.8.30-1_all.deb
 
 # fetch the updated package metadata (in particular, mysql-server)
 RUN apt-get update
@@ -44,10 +44,10 @@ RUN apt-get install -y mysql-server
 
 RUN apt-get -y install build-essential
 RUN apt-get -y install --reinstall python3-m2crypto python3-cryptography
-RUN apt-get -y install libxml2-dev libxmlsec1-dev swig
+RUN apt-get -y install libxml2-dev libxmlsec1-dev swig pkg-config
 RUN pip install pexpect
 
-RUN apt-get -y install unzip libffi-dev libssl-dev libmysqlclient-dev python3-mysqldb python3-dev libpython3-dev git ruby g++ curl
+RUN apt-get -y install unzip libffi-dev libssl-dev libmysqlclient-dev python3-mysqldb python3-dev libpython3-dev git g++ curl
 
 ADD . /app
 

README.md

@@ -55,11 +55,16 @@ You will need to set the `shell/python-su.sh` file to be executable. You can do
 To run your server in PyCharm:
 
  1. Make sure your Vagrant machine is running by going to **Tools > Vagrant > Up**
-    - If this is the first time you've built the VM, it can be time consuming.
-    - Our VMs are currently running Ubuntu 16.0.4 LTS, which is what the app deploys under as well.
- 2. Once the VM has built, click on the Run or Debug icons in the toolbar (upper-right corner of the PyCharm GUI)
-    - Your server will start and the PyCharm console should show all the logs and output from the system. 
-    - If you are running in debug, you can also use breakpoints to stop the execution and examine variables and code as it runs.
+  * If this is the first time you've built the VM, it can be time consuming.
+  * Our VMs are currently running Debian 12 (Bookworm) LTS, which is what the app deploys under as well.
+ 2. Once the VM is built, you will need to update the kernel headers and Guest Additions
+  * Kernel header update: `sudo apt-get -y install dkms build-essential linux-headers-$(uname -r)`
+    * NOTE: you may get a 'package not found' error here; if so, you'll need to look up the current header package for this install and use that instead.
+  * Guest Additions ISO mounting and installation: https://docs.bitnami.com/virtual-machine/faq/configuration/install-virtualbox-guest-additions/
+ 3. Next, set the `shell/python-su.sh` script to executable in the vagrant machine's command line with the command `chmod +x /home/vagrant/www/shell/python-su.sh`
+ 4. You can now click on the Run or Debug icons in the toolbar (upper-right corner of the PyCharm GUI)
+  * Your server will start and the PyCharm console should show all the logs and output from the system. 
+  * If you are running in debug, you can also use breakpoints to stop the execution and examine variables and code as it runs.
 
 ## Adding Python Dependencies
 
@@ -71,11 +76,11 @@ To update your existing python dependencies because of a change, or to pull down
 
  1. Click **Tools > Start SSH session...**
  2. Select the Vagrant VM Connection you set up
- 3. Type `cd www; sudo pip3 install -r requirements.txt --upgrade -t lib/`
+ 3. Type `cd www; sudo pip install -r requirements.txt --upgrade -t lib/`
 
 Or from the command line, you can do this by doing the following:
 
  1. Open a terminal in the project directory
  2. Type `vagrant ssh` to login to the virtual machine
  3. Change directory to the `www` directory (`/home/vagrant/www/` is the full path)
- 4. Run `pip3 install -r requirements.txt --upgrade -t lib/`
+ 4. Run `pip install -r requirements.txt --upgrade -t lib/`

Vagrantfile

@@ -17,8 +17,8 @@ Vagrant.configure(2) do |config|
      vb.customize ["modifyvm", :id, "--paravirtprovider", "default"]
   end
 
-  config.vm.box_version = "11.20241217.1"
-  config.vm.box = "debian/bullseye64"
+  config.vm.box = "debian/bookworm64"
+  config.vm.box_version = "12.20250126.1"
 
   # WebApp ports
   config.vm.network "forwarded_port", guest: 8086, host: 8086
@@ -38,10 +38,11 @@ Vagrant.configure(2) do |config|
   config.vm.provision :shell, inline: "sudo apt-get install dos2unix", :run => 'always'
   config.vm.provision :shell, inline: "dos2unix /home/vagrant/www/shell/*.sh", :run => 'always'
   config.vm.provision :shell, inline: "echo 'source /home/vagrant/www/shell/env.sh' > /etc/profile.d/sa-environment.sh", :run => 'always'
+  # This script will kill any further building if settings appear to be wrong
+  config.vm.provision "shell", path: 'shell/check-settings.sh', :run => 'always'
   config.vm.provision "shell", path: 'shell/install-deps.sh'
   # TODO: Adjust create and setup to check for database and run if it's not found so they can be set to always
   config.vm.provision "shell", path: 'shell/create-database.sh'
   config.vm.provision "shell", path: 'shell/database-setup.sh'
-  config.vm.provision "shell", path: 'shell/vagrant-start-server.sh', :run => 'always'
   config.vm.provision "shell", path: 'shell/vagrant-set-env.sh', :run => 'always'
 end

etl/etl.py

@@ -48,7 +48,7 @@
 from django.contrib.auth.models import User
 idc_superuser = User.objects.get(username="idc")
 
-logger = logging.getLogger('main_logger')
+logger = logging.getLogger(__name__)
 
 ATTR_SET = {}
 DISPLAY_VALS = {}

idc/demo_views.py

@@ -42,7 +42,7 @@
 from django.utils.html import escape
 
 debug = settings.DEBUG
-logger = logging.getLogger('main_logger')
+logger = logging.getLogger(__name__)
 
 BQ_ATTEMPT_MAX = 10
 WEBAPP_LOGIN_LOG_NAME = settings.WEBAPP_LOGIN_LOG_NAME

idc/domain_redirect_middleware.py

@@ -19,7 +19,7 @@
 from django.http import HttpResponsePermanentRedirect
 from django.conf import settings
 
-logger = logging.getLogger('main_logger')
+logger = logging.getLogger(__name__)
 
 
 class DomainRedirectMiddleware(object):

idc/metadata_utils.py

@@ -18,7 +18,7 @@
 from idc_collections.models import Attribute
 from django.conf import settings
 
-logger = logging.getLogger('main_logger')
+logger = logging.getLogger(__name__)
 
 
 def retTuple(x, order_docs):

idc/models.py

@@ -19,7 +19,7 @@
 
 import logging
 
-logger = logging.getLogger('main_logger')
+logger = logging.getLogger(__name__)
 
 
 class AppInfo(models.Model):

idc/settings.py

@@ -1,5 +1,5 @@
 ###
-# Copyright 2015-2023, Institute for Systems Biology
+# Copyright 2015-2025, Institute for Systems Biology
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -84,13 +84,14 @@
 DEFAULT_AUTO_FIELD = 'django.db.models.AutoField'
 
 GCLOUD_PROJECT_ID              = os.environ.get('GCLOUD_PROJECT_ID', '')
+GCLOUD_TOPICS_ID               = os.environ.get('GCLOUD_TOPICS_ID', GCLOUD_PROJECT_ID)
 GCLOUD_PROJECT_NUMBER          = os.environ.get('GCLOUD_PROJECT_NUMBER', '')
 BIGQUERY_PROJECT_ID            = os.environ.get('BIGQUERY_PROJECT_ID', GCLOUD_PROJECT_ID)
 BIGQUERY_DATA_PROJECT_ID       = os.environ.get('BIGQUERY_DATA_PROJECT_ID', GCLOUD_PROJECT_ID)
 BIGQUERY_USER_DATA_PROJECT_ID  = os.environ.get('BIGQUERY_USER_DATA_PROJECT_ID', GCLOUD_PROJECT_ID)
 BIGQUERY_USER_MANIFEST_DATASET = os.environ.get('BIGQUERY_USER_MANIFEST_DATASET', 'dev_user_dataset')
 BIGQUERY_USER_MANIFEST_TIMEOUT = int(os.environ.get('BIGQUERY_USER_MANIFEST_TIMEOUT', '7'))
-PUBSUB_USER_MANIFEST_TOPIC     = "projects/{}/topics/{}".format(GCLOUD_PROJECT_ID, os.environ.get('PUBSUB_USER_MANIFEST_TOPIC', 'user-manifest'))
+PUBSUB_USER_MANIFEST_TOPIC     = "projects/{}/topics/{}".format(GCLOUD_TOPICS_ID, os.environ.get('PUBSUB_USER_MANIFEST_TOPIC', 'user-manifest'))
 USER_MANIFESTS_FOLDER          = os.environ.get('USER_MANIFESTS_FOLDER', 'user-manifests')
 RESULT_BUCKET                  = os.environ.get('RESULT_BUCKET', 'idc-dev-files')
 
@@ -284,9 +285,10 @@
     'django.contrib.sessions.middleware.SessionMiddleware',
     'idc.checkreqsize_middleware.CheckReqSize',
     'django.middleware.csrf.CsrfViewMiddleware',
+    'django.contrib.messages.middleware.MessageMiddleware',
     'django.contrib.auth.middleware.AuthenticationMiddleware',
     'adminrestrict.middleware.AdminPagesRestrictMiddleware',
-    'django.contrib.messages.middleware.MessageMiddleware',
+    "allauth.account.middleware.AccountMiddleware",
     'idc.team_only_middleware.TeamOnly',
     # Uncomment the next line for simple clickjacking protection:
     'django.middleware.clickjacking.XFrameOptionsMiddleware',
@@ -311,7 +313,6 @@
     'django.contrib.humanize',
     'anymail',
     'idc',
-    'data_upload',
     'sharing',
     'cohorts',
     'idc_collections',
@@ -590,19 +591,28 @@
 DEFAULT_FROM_EMAIL = NOTIFICATION_EMAIL_FROM_ADDRESS
 SERVER_EMAIL = "[email protected]"
 
-GOOGLE_APPLICATION_CREDENTIALS  = join(dirname(__file__), '../{}{}'.format(SECURE_LOCAL_PATH,os.environ.get('GOOGLE_APPLICATION_CREDENTIALS', '')))
 OAUTH2_CLIENT_ID                = os.environ.get('OAUTH2_CLIENT_ID', '')
 OAUTH2_CLIENT_SECRET            = os.environ.get('OAUTH2_CLIENT_SECRET', '')
 
-if not exists(GOOGLE_APPLICATION_CREDENTIALS):
-    print("[ERROR] Google application credentials file wasn't found! Provided path: {}".format(GOOGLE_APPLICATION_CREDENTIALS))
-    exit(1)
+# WJRL 5/21/25: Pulling in the GAC test code now in use on ISB-CGC to support keyless deployment:
+#
+# Deployed systems retrieve credentials from the metadata server, but a local VM build must provide a credentials file
+# for some actions. CircleCI needs SA access but can make use of the deployment SA's key.
+GOOGLE_APPLICATION_CREDENTIALS = None
+
+if IS_DEV:
+    GOOGLE_APPLICATION_CREDENTIALS = os.environ.get('GOOGLE_APPLICATION_CREDENTIALS', '')
+elif IS_CI:
+    GOOGLE_APPLICATION_CREDENTIALS = "deployment.key.json"
+
+if not IS_APP_ENGINE:
+    if GOOGLE_APPLICATION_CREDENTIALS is not None and not exists(GOOGLE_APPLICATION_CREDENTIALS):
+        print("[ERROR] Google application credentials file wasn't found! Provided path: {}".format(GOOGLE_APPLICATION_CREDENTIALS))
+        exit(1)
+    print("[STATUS] GOOGLE_APPLICATION_CREDENTIALS: {}".format(GOOGLE_APPLICATION_CREDENTIALS))
+else:
+    print("[STATUS] AppEngine Flex detected--default credentials will be used.")
 
-#################################
-#   For NIH/eRA Commons login   #
-#################################
-GOOGLE_GROUP_ADMIN                      = os.environ.get('GOOGLE_GROUP_ADMIN', '')
-SUPERADMIN_FOR_REPORTS                  = os.environ.get('SUPERADMIN_FOR_REPORTS', '')
 
 
 ##############################
@@ -668,7 +678,7 @@
 
 
 # Explicitly check for known problems in descriptions and names provided by users
-DENYLIST_RE = r'((?i)<script>|(?i)</script>|!\[\]|!!\[\]|\[\]\[\".*\"\]|(?i)<iframe>|(?i)</iframe>)'
+DENYLIST_RE = r'(<script>|</script>|!\[\]|!!\[\]|\[\]\[\".*\"\]|<iframe>|</iframe>)'
 ATTRIBUTE_DISALLOW_RE = r'([^a-zA-Z0-9_])'
 
 if DEBUG and DEBUG_TOOLBAR and not IS_APP_ENGINE: