Skip to content

Commit 1e2ebfe

Browse files
martinburchellmartinburchell
authored
Merge pull request #99 from ucam-department-of-psychiatry/python310-dev
Update to Python 3.10, fix GitHub workflow and automated tests
2 parents d888b77 + df14632 commit 1e2ebfe

File tree

8 files changed

+323
-114
lines changed

8 files changed

+323
-114
lines changed

‎.github/workflows/im-build.yml‎

Lines changed: 23 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -8,12 +8,12 @@ jobs:
88
build:
99

1010
runs-on: ubuntu-latest
11-
11+
1212
services:
1313
# Label used to access the service container
1414
postgres:
15-
image: postgres:11
16-
env:
15+
image: postgres:14
16+
env:
1717
POSTGRES_USER: postgres
1818
POSTGRES_PASSWORD: postgres
1919
options: >-
@@ -22,24 +22,35 @@ jobs:
2222
--health-timeout 5s
2323
--health-retries 5
2424
ports:
25-
- 5432/tcp
26-
env:
27-
TESTMODEL_URL: http://localhost:8080/intermine-demo
28-
TESTMODEL_PATH: intermine-demo
25+
- 5432:5432
2926
steps:
30-
- uses: actions/checkout@v2
31-
- name: Set up python 3.7
32-
uses: actions/setup-python@v2
27+
- uses: actions/checkout@v4
28+
- name: Set up JDK 11
29+
uses: actions/setup-java@v2
30+
with:
31+
distribution: 'temurin'
32+
java-version: '11'
33+
- name: Set up python 3.10
34+
uses: actions/setup-python@v5
3335
with:
34-
python-version: '3.7'
36+
python-version: '3.10'
3537
- name: Install dependencies
3638
run: |
39+
set -euo pipefail
40+
sudo apt-get install libxml2-dev libxslt-dev python3-dev
3741
python -m pip install --upgrade pip
42+
pip install -U setuptools
3843
pip install -r requirements.txt
3944
- name: Install PostgreSQL client
4045
run: |
46+
set -euo pipefail
4147
sudo apt-get update -y
4248
sudo apt-get install -y libpq-dev postgresql-client
4349
sudo service postgresql start
4450
- name: Run unit tests
45-
run: ./config/ci/init-solr.sh && ./config/ci/init.sh && python setup.py test && python setup.py livetest
51+
run: |
52+
set -euo pipefail
53+
./config/ci/init-solr.sh ${GITHUB_WORKSPACE}
54+
./config/ci/init.sh ${GITHUB_WORKSPACE} http://localhost:8080/intermine-demo
55+
python setup.py test
56+
python setup.py livetest

‎.gitignore‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,3 +8,10 @@ intermine-docs-1.00.03.zip
88
build
99
MANIFEST
1010
.idea
11+
12+
# CI
13+
14+
server/
15+
16+
solr-*.tgz
17+
solr-*/

‎config/ci/init-solr.sh‎

Lines changed: 43 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,46 @@
1+
#!/bin/bash
2+
13
# set up solr for testmine
24
# testmine's setup script populates these empty indexes
35

4-
wget http://archive.apache.org/dist/lucene/solr/8.6.2/solr-8.6.2.tgz
5-
tar xzf solr-8.6.2.tgz && ./solr-8.6.2/bin/solr start
6-
./solr-8.6.2/bin/solr create -c intermine-search
7-
./solr-8.6.2/bin/solr create -c intermine-autocomplete
6+
set -euo pipefail
7+
8+
if [ "$#" != "1" ]; then
9+
echo "Usage: $0 <workspace_dir>"
10+
exit 2
11+
fi
12+
13+
WORKSPACE_DIR=$1
14+
15+
cd "$WORKSPACE_DIR"
16+
17+
SOLR_VERSION=8.6.2
18+
SOLR_PACKAGE=solr-${SOLR_VERSION}.tgz
19+
SOLR_DIR=solr-${SOLR_VERSION}
20+
SOLR=${SOLR_DIR}/bin/solr
21+
22+
create_solr_core() {
23+
local core_name=$1
24+
25+
local status
26+
27+
status=$(curl -s "http://localhost:8983/solr/admin/cores?action=STATUS&core=${core_name}" | jq --arg core_name "${core_name}" '.status[$core_name]')
28+
29+
if [ "$status" = "{}" ]; then
30+
${SOLR} create -c "${core_name}"
31+
else
32+
echo "Solr core ${core_name} already exists"
33+
fi
34+
}
35+
36+
if [ ! -d $SOLR_DIR ]; then
37+
if [ ! -f $SOLR_PACKAGE ]; then
38+
wget http://archive.apache.org/dist/lucene/solr/${SOLR_VERSION}/${SOLR_PACKAGE}
39+
fi
40+
41+
tar xzf $SOLR_PACKAGE
42+
fi
43+
44+
${SOLR} restart
45+
create_solr_core intermine-search
46+
create_solr_core intermine-autocomplete

‎config/ci/init.sh‎

Lines changed: 36 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -1,44 +1,61 @@
11
#!/bin/bash
22

3-
set -e
3+
set -euo pipefail
44

5-
if [ -z $(which wget) ]; then
6-
# use curl
7-
GET='curl'
8-
else
9-
GET='wget -O -'
5+
if [ "$#" != "2" ]; then
6+
echo "Usage: $0 <workspace_dir> <testmodel_url>"
7+
exit 2
108
fi
119

10+
WORKSPACE_DIR=$1
11+
TESTMODEL_URL=$2
12+
13+
INTERMINE_DIR=${WORKSPACE_DIR}/server
14+
TESTMINE_DIR=${INTERMINE_DIR}/testmine
15+
16+
cd "${WORKSPACE_DIR}"
17+
1218
# Pull in the server code.
13-
git clone --single-branch --depth 1 https://github.com/intermine/intermine.git server
19+
20+
if [ ! -d "${INTERMINE_DIR}" ]; then
21+
git clone --single-branch --depth 1 -b dev https://github.com/ucam-department-of-psychiatry/intermine.git "${INTERMINE_DIR}"
22+
fi
1423

1524
export PSQL_USER=test
1625
export PSQL_PWD=test
26+
export PSQL_HOST=localhost
27+
export PGPASSWORD=${PGPASSWORD:-postgres}
1728
export KEYSTORE=${PWD}/keystore.jks
1829

1930
echo "#---> Running unit tests"
2031

21-
sudo -u postgres createuser test
22-
sudo -u postgres psql -c "alter user test with encrypted password 'test';"
32+
sudo -E -u postgres dropdb -h "$PSQL_HOST" --if-exists intermine-demo
33+
sudo -E -u postgres dropdb -h "$PSQL_HOST" --if-exists userprofile-demo
34+
35+
sudo -E -u postgres dropuser -h "${PSQL_HOST}" --if-exists test
36+
sudo -E -u postgres createuser -h "${PSQL_HOST}" test
37+
sudo -E -u postgres psql -h "${PSQL_HOST}" -c "alter user test with encrypted password 'test';"
2338

2439
# Set up properties
25-
PROPDIR=$HOME/.intermine
26-
TESTMODEL_PROPS=$PROPDIR/testmodel.properties
27-
SED_SCRIPT='s/PSQL_USER/test/'
40+
PROPDIR=${HOME}/.intermine
41+
TESTMODEL_PROPS=${PROPDIR}/testmodel.properties
42+
43+
mkdir -p "${PROPDIR}"
2844

29-
mkdir -p $PROPDIR
45+
echo "#--- creating ${TESTMODEL_PROPS}"
46+
cp "${INTERMINE_DIR}"/config/testmodel.properties "${TESTMODEL_PROPS}"
47+
sed -i -e "s/PSQL_HOST/${PSQL_HOST}/" "$TESTMODEL_PROPS"
48+
sed -i -e "s/PSQL_USER/${PSQL_USER}/" "$TESTMODEL_PROPS"
49+
sed -i -e "s/PSQL_PWD/${PSQL_PWD}/" "$TESTMODEL_PROPS"
3050

31-
echo "#--- creating $TESTMODEL_PROPS"
32-
cp server/config/testmodel.properties $TESTMODEL_PROPS
33-
sed -i -e $SED_SCRIPT $TESTMODEL_PROPS
3451

3552
# We will need a fully operational web-application
3653
echo '#---> Building and releasing web application to test against'
37-
(cd server/testmine && ./setup.sh)
54+
(cd "${TESTMINE_DIR}" && ./setup.sh "${INTERMINE_DIR}")
3855
# Travis is so slow
3956
sleep 90 # let webapp startup
4057

4158
# Warm up the keyword search by requesting results, but ignoring the results
42-
$GET "$TESTMODEL_URL/service/search" > /dev/null
59+
wget -O - "${TESTMODEL_URL}/service/search" > /dev/null
4360
# Start any list upgrades
44-
$GET "$TESTMODEL_URL/service/lists?token=test-user-token" > /dev/null
61+
wget -O - "${TESTMODEL_URL}/service/lists?token=test-user-token" > /dev/null

‎intermine/results.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -603,7 +603,7 @@ def __init__(self, credentials=None, token=None):
603603
self.token = token
604604
if credentials and len(credentials) == 2:
605605
encoded = '{0}:{1}'.format(*credentials).encode('utf8')
606-
base64string = 'Basic {0}'.format(base64.encodestring(encoded)[:-1].decode('ascii'))
606+
base64string = 'Basic {0}'.format(base64.encodebytes(encoded)[:-1].decode('ascii'))
607607
self.auth_header = base64string
608608
self.using_authentication = True
609609
elif self.token is not None:

‎intermine/webservice.py‎

Lines changed: 4 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -5,16 +5,10 @@
55

66
import requests
77

8-
try:
9-
from urlparse import urlparse
10-
from UserDict import DictMixin
11-
from urllib import urlopen
12-
from urllib import urlencode
13-
except ImportError:
14-
from urllib.parse import urlparse
15-
from urllib.parse import urlencode
16-
from collections import MutableMapping as DictMixin
17-
from urllib.request import urlopen
8+
from urllib.parse import urlparse
9+
from urllib.parse import urlencode
10+
from collections.abc import MutableMapping as DictMixin
11+
from urllib.request import urlopen
1812

1913
try:
2014
import simplejson as json # Prefer this as it is faster

‎requirements.txt‎

Lines changed: 17 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,17 @@
1-
certifi==2020.4.5.1
2-
chardet==3.0.4
3-
cycler==0.10.0
4-
Cython==0.29.19
5-
idna==2.9
6-
kiwisolver==1.2.0
7-
lxml==4.5.1
8-
matplotlib==3.2.1
9-
numpy==1.18.4
10-
pandas==1.0.3
11-
pyparsing==2.4.7
12-
python-dateutil==2.8.1
13-
pytz==2020.1
14-
requests==2.23.0
15-
simplejson==3.17.0
16-
six==1.15.0
17-
urllib3==1.25.9
1+
certifi==2024.8.30
2+
chardet==5.2.0
3+
cycler==0.12.1
4+
Cython==3.0.11
5+
idna==3.8
6+
kiwisolver==1.4.5
7+
lxml==5.3.0
8+
matplotlib==3.9.2
9+
numpy==2.1.0
10+
pandas==2.2.2
11+
pyparsing==3.1.4
12+
python-dateutil==2.9.0
13+
pytz==2024.1
14+
requests==2.32.4
15+
simplejson==3.19.3
16+
six==1.16.0
17+
urllib3==2.5.0

0 commit comments

Comments
 (0)