Skip to content

Commit 98db63a

Browse files
jacobfilikjacobfilik
authored
Initial prototype
* basic inital structure for DB and api * more orca features added to db and api * add very rough initial client for molecule api * simple simulation viewing page * start to add crystal structures * more work on submit ui, start backend job submitter * more features added, added msw * add initial fdmnes pages * initial submission working * Convert to react query * further improvements, plot orca results * add initial backend unit test and CI * add tests and ci for client * switch client ci to use ubuntu latest * use pnpm 10 in CI
1 parent ef3a56f commit 98db63a

File tree

94 files changed

+11756
-2
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

94 files changed

+11756
-2
lines changed

‎.github/workflows/ci-backend.yml‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
# This workflow will install Python dependencies, run tests and lint with a single version of Python
2+
# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
3+
4+
name: Backend CI
5+
6+
on:
7+
push:
8+
branches: [ main ]
9+
pull_request:
10+
branches: [ main ]
11+
12+
# Allows you to run this workflow manually from the Actions tab
13+
workflow_dispatch:
14+
15+
jobs:
16+
build:
17+
runs-on: ubuntu-latest
18+
defaults:
19+
run:
20+
working-directory: ./web-conexs-api
21+
strategy:
22+
matrix:
23+
python-version: ["3.10", "3.11"]
24+
25+
steps:
26+
- uses: actions/checkout@v4
27+
- name: Set up Python
28+
uses: actions/setup-python@v4
29+
with:
30+
python-version: ${{ matrix.python-version }}
31+
- name: Install dependencies
32+
run: |
33+
pip install .[dev]
34+
- name: Lint
35+
run: tox -e pre-commit
36+
- name: Test with pytest
37+
run: |
38+
python3 -m pytest

‎.github/workflows/ci-frontend.yml‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
name: frontend ci
2+
3+
on:
4+
push:
5+
branches: [ main ]
6+
pull_request:
7+
branches: [ main ]
8+
9+
jobs:
10+
build:
11+
runs-on: ubuntu-latest
12+
defaults:
13+
run:
14+
working-directory: ./web-conexs-client
15+
strategy:
16+
matrix:
17+
node-version: [23]
18+
steps:
19+
- uses: actions/checkout@v4
20+
- uses: pnpm/action-setup@v4
21+
with:
22+
version: 10
23+
- name: Use Node.js ${{ matrix.node-version }}
24+
uses: actions/setup-node@v4
25+
with:
26+
node-version: ${{ matrix.node-version }}
27+
cache: 'pnpm'
28+
cache-dependency-path: '**/pnpm-lock.yaml'
29+
- name: Install dependencies
30+
run: pnpm install --frozen-lockfile
31+
- name: Run test
32+
run: pnpm test

‎README.md‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,5 +4,3 @@ Simulation-as-a-Service platform for X-ray Absorption Spectroscopy.
44

55
https://www.diamond.ac.uk/Science/CommunitiesEngagement/CONEXS.html
66
https://research.ncl.ac.uk/conexs/
7-
8-

‎database/tables.sql‎

Lines changed: 171 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,171 @@
1+
\c webconexsdb
2+
3+
SET role webconexsadmin;
4+
5+
CREATE TABLE person (
6+
id INTEGER PRIMARY KEY GENERATED ALWAYS AS IDENTITY,
7+
identifier TEXT NOT NULL,
8+
admin BOOLEAN NOT NULL DEFAULT FALSE,
9+
UNIQUE (identifier)
10+
);
11+
12+
COMMENT ON TABLE person IS 'Table to store unique identifier of user';
13+
14+
CREATE TABLE simulation_type (
15+
id INTEGER PRIMARY KEY,
16+
type TEXT NOT NULL
17+
);
18+
19+
COMMENT ON TABLE simulation_type IS 'Table to store type of simulation job';
20+
21+
INSERT INTO "simulation_type" VALUES(1,'ORCA');
22+
INSERT INTO "simulation_type" VALUES(2,'FDMNES');
23+
INSERT INTO "simulation_type" VALUES(3,'Quantum ESPRESSO');
24+
25+
26+
CREATE TYPE simulation_status_enum AS ENUM('requested', 'submitted', 'running', 'completed', 'failed', 'error');
27+
28+
CREATE TABLE simulation (
29+
id INTEGER PRIMARY KEY GENERATED ALWAYS AS IDENTITY,
30+
simulation_type_id INTEGER NOT NULL,
31+
person_id INTEGER NOT NULL,
32+
working_directory TEXT,
33+
n_cores INTEGER NOT NULL DEFAULT 4,
34+
memory INTEGER NOT NULL DEFAULT 32,
35+
status simulation_status_enum DEFAULT 'requested',
36+
message TEXT,
37+
job_id INTEGER,
38+
request_date TIMESTAMP DEFAULT current_timestamp,
39+
submission_date TIMESTAMP,
40+
completion_date TIMESTAMP,
41+
FOREIGN KEY(simulation_type_id) REFERENCES simulation_type (id),
42+
FOREIGN KEY(person_id) REFERENCES person (id),
43+
constraint simulation_altpk unique(id,simulation_type_id)
44+
);
45+
46+
COMMENT ON TABLE simulation IS 'Base table for simulations';
47+
48+
--CREATE OR REPLACE FUNCTION notify_new_simulation() RETURNS trigger AS $$
49+
--BEGIN
50+
-- PERFORM pg_notify('simulation_notification', NEW.id::text);
51+
-- RETURN NEW;
52+
--END;
53+
--$$ LANGUAGE plpgsql;
54+
55+
--CREATE TRIGGER simulation_notify_trigger
56+
--AFTER INSERT ON simulation
57+
--FOR EACH ROW EXECUTE PROCEDURE notify_new_simulation();
58+
59+
60+
CREATE TABLE molecular_structure (
61+
id INTEGER PRIMARY KEY GENERATED ALWAYS AS IDENTITY,
62+
label TEXT,
63+
structure TEXT,
64+
person_id INTEGER NOT NULL
65+
66+
);
67+
68+
COMMENT ON TABLE molecular_structure IS 'Table to hold molecular structures';
69+
70+
71+
CREATE TYPE ibrav_enum AS ENUM('1', '2', '3','-3', '4', '5','-5', '6', '7', '8', '9','-9','91', '10', '11', '12','-12', '13','-13', '14');
72+
73+
74+
CREATE TABLE crystal_structure (
75+
id INTEGER PRIMARY KEY GENERATED ALWAYS AS IDENTITY,
76+
label TEXT,
77+
structure TEXT NOT NULL,
78+
a NUMERIC NOT NULL,
79+
b NUMERIC NOT NULL,
80+
c NUMERIC NOT NULL,
81+
alpha NUMERIC NOT NULL,
82+
beta NUMERIC NOT NULL,
83+
gamma NUMERIC NOT NULL,
84+
ibrav ibrav_enum NOT NULL,
85+
person_id INTEGER NOT NULL
86+
);
87+
88+
COMMENT ON TABLE molecular_structure IS 'Table to hold molecular structures';
89+
90+
CREATE TYPE orca_calculation_enum AS ENUM('xas', 'xes', 'opt');
91+
92+
CREATE TYPE orca_solvent_enum AS ENUM('Water','Acetone', 'Acetonitrile', 'Ammonia', 'Benzene', 'CCl4', 'CH2Cl2', 'Chloroform', 'Cyclohexane', 'DMF', 'DMSO', 'Ethanol', 'Hexane', 'Methanol', 'Octanol', 'Pyridine', 'THF','Toluene');
93+
94+
CREATE TABLE orca_simulation (
95+
simulation_id INTEGER PRIMARY KEY,
96+
simulation_type_id INTEGER GENERATED ALWAYS AS (1) STORED,
97+
calculation_type orca_calculation_enum,
98+
molecular_structure_id INTEGER NOT NULL,
99+
memory_per_core INTEGER NOT NULL,
100+
functional TEXT NOT NULL,
101+
basis_set TEXT NOT NULL,
102+
charge INTEGER NOT NULL,
103+
multiplicity INTEGER NOT NULL,
104+
solvent orca_solvent_enum,
105+
orb_win_0_start INTEGER DEFAULT 0,
106+
orb_win_0_stop INTEGER DEFAULT 0,
107+
orb_win_1_start INTEGER DEFAULT 0,
108+
orb_win_1_stop INTEGER DEFAULT 0,
109+
FOREIGN KEY(molecular_structure_id) REFERENCES molecular_structure (id),
110+
FOREIGN KEY(simulation_id, simulation_type_id) REFERENCES simulation (id,simulation_type_id)
111+
);
112+
113+
COMMENT ON TABLE orca_simulation IS 'Specific information for an orca simulation';
114+
115+
CREATE TYPE orca_spectrum_enum AS ENUM('abs', 'absq', 'xes', 'xesq');
116+
117+
CREATE TABLE orca_simulation_spectrum(
118+
id INTEGER PRIMARY KEY GENERATED ALWAYS AS IDENTITY,
119+
simulation_id INTEGER NOT NULL,
120+
spectrum_type orca_spectrum_enum,
121+
start INTEGER NOT NULL,
122+
stop INTEGER NOT NULL,
123+
broadening NUMERIC NOT NULL,
124+
working_dir TEXT NOT NULL,
125+
126+
FOREIGN KEY(simulation_id) REFERENCES orca_simulation (simulation_id)
127+
128+
);
129+
130+
CREATE TYPE edge_enum AS ENUM('k', 'l1', 'l2', 'l3', 'm1', 'm2', 'm3', 'm4', 'm5');
131+
132+
CREATE TYPE structure_enum AS ENUM('crystal', 'molecule');
133+
134+
CREATE TABLE fdmnes_simulation (
135+
simulation_id INTEGER PRIMARY KEY,
136+
simulation_type_id INTEGER GENERATED ALWAYS AS (2) STORED,
137+
crystal_structure_id INTEGER NOT NULL,
138+
element INTEGER NOT NULL,
139+
edge edge_enum NOT NULL,
140+
greens_approach BOOLEAN NOT NULL,
141+
structure_type structure_enum NOT NULL,
142+
143+
FOREIGN KEY(crystal_structure_id) REFERENCES crystal_structure (id),
144+
FOREIGN KEY(simulation_id, simulation_type_id) REFERENCES simulation (id,simulation_type_id)
145+
);
146+
147+
COMMENT ON TABLE fdmnes_simulation IS 'Specific information for a FDMNES simulation';
148+
149+
CREATE TABLE qe_simulation (
150+
simulation_id INTEGER PRIMARY KEY,
151+
simulation_type_id INTEGER GENERATED ALWAYS AS (3) STORED,
152+
FOREIGN KEY(simulation_id, simulation_type_id) REFERENCES simulation (id,simulation_type_id)
153+
);
154+
155+
COMMENT ON TABLE qe_simulation IS 'Specific information for a Quantum ESPRESSO simulation';
156+
157+
INSERT INTO person(identifier) VALUES('test_user');
158+
159+
INSERT INTO molecular_structure(label, person_id, structure) VALUES('Water', 1, 'H 0.7493682 0.0000000 0.4424329
160+
O 0.0000000 0.0000000 -0.1653507
161+
H -0.7493682 0.0000000 0.4424329');
162+
163+
164+
INSERT INTO crystal_structure(label, person_id,a,b,c,alpha,beta,gamma,ibrav,structure) VALUES('Silver',1,4.1043564,4.1043564,4.1043564,90,90,90,'2','Ag 0.0 0.0 0.0
165+
Ag 0.5 0.5 0.0
166+
Ag 0.5 0.0 0.5
167+
Ag 0.0 0.5 0.5');
168+
169+
170+
INSERT INTO crystal_structure(label, person_id,a,b,c,alpha,beta,gamma,ibrav,structure) VALUES('KCl',1,6.28,6.28,6.28,90,90,90,'2','K 0.0 0.0 0.0
171+
Cl 0.5 0.0 0.0');

‎docker-compose.yaml‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
services:
2+
postgres:
3+
image: postgres:12
4+
restart: always
5+
user: root
6+
environment:
7+
POSTGRES_USER: webconexsadmin
8+
POSTGRES_PASSWORD: password
9+
POSTGRES_DB: webconexsdb
10+
ports:
11+
- '5432:5432'
12+
volumes:
13+
- postgres-data:/var/lib/postgresql/data
14+
- ./database/tables.sql:/docker-entrypoint-initdb.d/create_tables.sql
15+
16+
volumes:
17+
postgres-data:

‎web-conexs-api/.devcontainer/devcontainer.json‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
// For format details, see https://containers.dev/implementors/json_reference/
2+
{
3+
"name": "Python 3 Developer Container",
4+
"build": {
5+
"dockerfile": "../Dockerfile",
6+
"target": "developer"
7+
},
8+
"remoteEnv": {
9+
// Allow X11 apps to run inside the container
10+
"DISPLAY": "${localEnv:DISPLAY}"
11+
},
12+
"containerEnv": {"CONEXS_ROOT_DIR" : "${localEnv:ROOT_DIR}",
13+
"CONEXS_CLUSTER_ROOT_DIR" : "${localEnv:ROOT_DIR}",
14+
"SLURM_USER" : "${localEnv:USER}",
15+
"SLURM_API" : "${localEnv:SLURM_API}",
16+
"SLURM_PARTITION": "${localEnv:SLURM_PARTITION}",
17+
"ORCA_IMAGE": "${localEnv:ORCA_IMAGE}",
18+
"FDMNES_IMAGE": "${localEnv:FDMNES_IMAGE}",
19+
"CONTAINER_IMAGE_DIR": "${localEnv:SLURM_PARTITION}",
20+
"SLURM_TOKEN": "${localEnv:SLURM_TOKEN}",
21+
"POSTGRESURL": "${localEnv:POSTGRESURL}" },
22+
23+
"mounts" :["source=/dls/tmp/vdp96513/conexs,target=/dls/tmp/vdp96513/conexs,type=bind,consistency=cached"],
24+
"customizations": {
25+
"vscode": {
26+
// Set *default* container specific settings.json values on container create.
27+
"settings": {
28+
"python.defaultInterpreterPath": "/venv/bin/python"
29+
},
30+
// Add the IDs of extensions you want installed when the container is created.
31+
"extensions": [
32+
"ms-python.python",
33+
"github.vscode-github-actions",
34+
"tamasfe.even-better-toml",
35+
"redhat.vscode-yaml",
36+
"ryanluker.vscode-coverage-gutters",
37+
"charliermarsh.ruff",
38+
"ms-azuretools.vscode-docker"
39+
]
40+
}
41+
},
42+
"features": {
43+
// add in eternal history and other bash features
44+
"ghcr.io/diamondlightsource/devcontainer-features/bash-config:1": {}
45+
},
46+
// Create the config folder for the bash-config feature
47+
"initializeCommand": "mkdir -p ${localEnv:HOME}/.config/bash-config",
48+
"runArgs": [
49+
// Allow the container to access the host X11 display and EPICS CA
50+
"--net=host",
51+
// Make sure SELinux does not disable with access to host filesystems like tmp
52+
"--security-opt=label=disable"
53+
],
54+
// After the container is created, install the python project in editable form
55+
"postCreateCommand": "pip install $([ -f dev-requirements.txt ] && echo '-c dev-requirements.txt') -e '.[dev]' && pre-commit install"
56+
}

‎web-conexs-api/.gitignore‎

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
# Byte-compiled / optimized / DLL files
2+
__pycache__/
3+
*.py[cod]
4+
5+
# C extensions
6+
*.so
7+
8+
# Distribution / packaging
9+
.Python
10+
env/
11+
build/
12+
develop-eggs/
13+
dist/
14+
downloads/
15+
eggs/
16+
.eggs/
17+
lib/
18+
lib64/
19+
parts/
20+
sdist/
21+
var/
22+
*.egg-info/
23+
.installed.cfg
24+
*.egg
25+
**/_version.py
26+
27+
# PyInstaller
28+
# Usually these files are written by a python script from a template
29+
# before PyInstaller builds the exe, so as to inject date/other infos into it.
30+
*.manifest
31+
*.spec
32+
33+
# Installer logs
34+
pip-log.txt
35+
pip-delete-this-directory.txt
36+
37+
# Unit test / coverage reports
38+
htmlcov/
39+
.tox/
40+
.coverage
41+
.coverage.*
42+
.cache
43+
nosetests.xml
44+
coverage.xml
45+
cov.xml
46+
.pytest_cache/
47+
.mypy_cache/
48+
49+
# Translations
50+
*.mo
51+
*.pot
52+
53+
# Django stuff:
54+
*.log
55+
56+
# Sphinx documentation
57+
docs/_build/
58+
docs/_api
59+
60+
# PyBuilder
61+
target/
62+
63+
# likely venv names
64+
.venv*
65+
venv*
66+
67+
# further build artifacts
68+
lockfiles/
69+
70+
# ruff cache
71+
.ruff_cache/
72+
73+
#environment files
74+
*.env

0 commit comments

Comments
 (0)