Skip to content

Commit a3386ed

Browse files
committed
WebAPI for administraion of collections
1 parent e125819 commit a3386ed

File tree

12 files changed

+751
-156
lines changed

12 files changed

+751
-156
lines changed

app.py

Lines changed: 72 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -1,25 +1,21 @@
11
import os
2-
from flask import Flask, render_template, redirect, request, make_response, url_for, jsonify
2+
from flask import Flask, render_template, redirect, request, make_response, url_for, jsonify, Response
33
from waitress import serve
44
import argparse as AP
55
import requests
66
import yaml
77
import hashlib
88
import logging
9+
from pathlib import Path
910
from urllib.parse import urlparse
1011

11-
os.makedirs('logs',exist_ok=True)
12-
logfile = 'logs/app.log'
13-
14-
if os.path.exists(logfile): os.remove(logfile)
15-
logging.basicConfig(filename=logfile, level=logging.INFO)
16-
logger = logging.getLogger(__name__)
12+
LOG_FILE = 'app.log'
13+
IMPORTER_URL ='http://importer:5020'
14+
FUSEKI_URL ='http://fuseki:3030'
1715

1816

19-
sparql_url = 'http://fuseki:3030/n4o'
20-
def importer_url(coll):
21-
return f'http://importer:5020/collection/{coll}'
22-
17+
logging.basicConfig(filename=LOG_FILE, level=logging.INFO)
18+
logger = logging.getLogger(__name__)
2319

2420
app = Flask(__name__, template_folder='templates', static_folder='static', static_url_path='/assets')
2521
app.secret_key = 'your_secret_key' # Replace with a strong secret key
@@ -39,31 +35,28 @@ def pw_hash(pw_str):
3935
return hashlib.md5(pw_str.encode()).hexdigest()
4036

4137

42-
def is_RDF_suffix(suffix: str):
43-
'''Check if the file suffix is a valid RDF format'''
44-
return suffix.lower() in ['.ttl', '.nt', '.nq', '.jsonld', '.json', '.rdf', '.xml']
38+
def get_page(page, title=""):
39+
'''Render the home page'''
40+
if 'username' in request.cookies:
41+
return render_template(page, title=title)
42+
else:
43+
return redirect(url_for('login'))
4544

4645

47-
def import_file(storage_file, collection='default'):
48-
'''Import a file into the RDF store'''
49-
if collection:
50-
files = {'file': (storage_file.filename, storage_file, 'text/plain')}
51-
response = requests.post(sparql_url, files=files, params={'graph': f'n4o:{collection}'})
52-
return (f'Importing {storage_file.filename} into {collection} - Ok\nanswer={response.text}', None)
46+
@app.route('/')
47+
@app.route('/collections')
48+
def home():
49+
return get_page('collections.html', "N4O-KG: Collections")
5350

5451

55-
@app.route('/info')
56-
def info():
57-
'''Display information about the RDF store and users'''
58-
return f'Info: SPARQL = {sparql_url}'
52+
@app.route('/terminologies')
53+
def terminologies():
54+
return get_page('terminologies.html', "N4O-KG: Terminologies")
5955

60-
@app.route('/')
61-
def home():
62-
'''Render the home page'''
63-
if 'username' in request.cookies:
64-
return render_template('index.html', conn=jsonify('http://localhost:3030/n4o').json)
65-
else:
66-
return redirect(url_for('login'))
56+
57+
@app.route('/mappings')
58+
def mappings():
59+
return get_page('mappings.html', 'N4O-KG: Mappings')
6760

6861

6962
@app.route('/login', methods=['GET', 'POST'])
@@ -89,6 +82,53 @@ def logout():
8982
response.delete_cookie('username')
9083
return response
9184

85+
86+
@app.route('/fuseki', methods=['post'])
87+
def fuseki():
88+
if data := request.json.get('data'):
89+
res = requests.post(f'{FUSEKI_URL}/n4o?query={data}')
90+
return jsonify(res.text), res.status_code
91+
92+
93+
@app.route('/importer/<path:subpath>', methods=["GET", "POST", "PUT", "DELETE"])
94+
def importer(subpath):
95+
'''Forwards requests to the importer service'''
96+
res = requests.request(
97+
method=request.method,
98+
url=f'{IMPORTER_URL}/{subpath}', # Forward to importer service
99+
headers={k: v for k, v in request.headers if k.lower() != 'host'}, # Exclude 'host' header
100+
data=request.get_data(),
101+
cookies=request.cookies,
102+
json=request.get_json(silent=True),
103+
allow_redirects=False,
104+
)
105+
106+
excluded_headers = ['content-encoding', 'content-length', 'transfer-encoding', 'connection']
107+
headers = [(k, v) for k, v in res.raw.headers.items() if k.lower() not in excluded_headers]
108+
109+
response = Response(res.content, res.status_code, headers)
110+
return response
111+
112+
113+
@app.route('/postCollectionData', methods=['POST'])
114+
def postCollectionData():
115+
'''Posts new collection data. Recent data will be deleted.'''
116+
if uri := request.json.get('uri'):
117+
index = urlparse(uri).path.strip('/').split('/')[-1]
118+
if data := request.json['data']:
119+
# Copy data to file, then call importer services receive and add
120+
suffix = request.json.get("suffix", "ttl")
121+
import_file = f'import_{index}.{suffix}'
122+
Path(f'./data').mkdir(exist_ok=True)
123+
Path(f'./data/{import_file}').write_text(data)
124+
service = f'{IMPORTER_URL}/collection/{index}'
125+
res = requests.post(f'{service}/receive?from={import_file}')
126+
res = requests.post(f'{service}/load')
127+
#res = requests.post(f'{service}/add')
128+
return res.text, res.status_code
129+
return jsonify(message='No data or collection index provided')
130+
131+
92132
def read_yaml(fname):
93133
'''Read a YAML file and return the data'''
94134
if os.path.exists(fname):
@@ -104,13 +144,10 @@ def read_yaml(fname):
104144
args = parser.parse_args()
105145
opts = {"port": args.port}
106146

107-
if config_data := read_yaml(args.config):
108-
sparql_url = config_data["fuseki-server"]["uri"]
109-
110147
user_data = read_yaml('users.yaml')
111148
if not user_data:
112149
user_data = read_yaml('admin.yaml')
113-
150+
114151
if user_data:
115152
users = user_data['users']
116153
else:

static/collections.js

Lines changed: 247 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,247 @@
1+
const { log } = require("node:console");
2+
3+
/// Ensure collection has mandatory fields
4+
function fixCollection(collection) {
5+
let coll = collection
6+
if (!('type' in coll)) { coll.type = []; }
7+
if (!('db' in coll)) { coll.db = ''; }
8+
if (!('url' in coll)) { coll.url = ''; }
9+
if (!('access' in coll)) { coll.access = []; }
10+
return coll;
11+
};
12+
13+
/// Returns an URL for collections
14+
function collection_url(s="") {
15+
return `/importer/collection/${s}`;
16+
}
17+
18+
/// Get list of collections from the KG
19+
function getCollections() {
20+
const req = { method: 'GET', headers: { 'Accept': 'application/json, text/plain, */*', 'Content-Type': 'application/json' }};
21+
fetch(collection_url(), req)
22+
.then(response => response.json())
23+
.then(js => {
24+
app.collections = js.map(d => fixCollection(d));
25+
app.collections.sort((a, b) => a.id - b.id);
26+
})
27+
.catch(err => console.error(err))
28+
.finally(() => { console.log('fetch collections done'); });
29+
}
30+
31+
function makeCollection(id_str, collection_tpl) {
32+
const collection = collection_tpl ? { ...collection_tpl } : {
33+
type: [],
34+
access: [],
35+
partOf: [],
36+
url: '',
37+
db: '',
38+
license: '',
39+
};
40+
if (collection_tpl) {
41+
collection.id = id_str;
42+
collection.name = `New Collection ${id_str}`;
43+
collection.uri = `https://graph.nfdi4objects.net/collection/${id_str}`;
44+
}
45+
return collection
46+
}
47+
48+
function showProgess(on = false) {
49+
document.getElementById("SP1").style.display = on ? "inline-block" : "none";
50+
}
51+
52+
/// Import TTL data into the KG
53+
function postCollectionData(uri, data, suffix) {
54+
showProgess(true);
55+
var req = {
56+
method: 'POST',
57+
headers: {
58+
'Accept': 'application/json, text/plain, */*', 'Content-Type': 'application/json'
59+
},
60+
body: JSON.stringify({ "data": data, "uri": uri, suffix: suffix })
61+
}
62+
fetch('/postCollectionData', req)
63+
.then(response => response.json())
64+
.then(json => console.log(json))
65+
.catch(err => console.error(err))
66+
.finally(() => {
67+
showProgess(false);
68+
});
69+
}
70+
71+
function makeAppData() {
72+
return {
73+
data() {
74+
return {
75+
collections: [],
76+
displayCollection: null,
77+
collectionInfo: 'No info.',
78+
rdfFiles: [],
79+
}
80+
},
81+
delimiters: ["${", "}$"], // for global
82+
compilerOptions: { delimiters: ["${", "}$"] }, // for standalone
83+
methods: {
84+
selectCollection(c_id) {
85+
this.displayCollection = this.collections.find(c => c.id === c_id);
86+
},
87+
88+
delCollection() {
89+
/// Remove a type from the collection
90+
let N = this.displayCollection ? this.displayCollection.id :this.collections.length
91+
let id = prompt('Enter the index of the collection to delete:', N);
92+
id = parseInt(id) - 1;
93+
if (!isNaN(id) && id >= 0 || id < N) {
94+
let q = this.collections[id];
95+
//console.log(q.id)
96+
fetch(collection_url(q.id), {
97+
method: 'DELETE',
98+
headers: { "Content-Type": "application/json", }
99+
})
100+
.then(response => response.json())
101+
.then(data => {
102+
console.log(data);
103+
this.collections.splice(id, 1)
104+
this.displayCollection = null
105+
})
106+
.catch(err => alert(err))
107+
.finally(() => { console.log(`collection ${id} deleted`); });
108+
}
109+
110+
},
111+
112+
addCollection() {
113+
/// Add a new collection
114+
let new_id = 1;
115+
let N = this.collections.length;
116+
if (N > 0) {
117+
new_id = Math.max(...this.collections.map(c => c.id)) + 1;
118+
}
119+
let lastCollection = N > 0 ? this.collections[N - 1] : null;
120+
const new_collection = makeCollection(new_id.toString(), lastCollection)
121+
122+
fetch(collection_url(), {
123+
method: 'POST',
124+
headers: { "Content-Type": "application/json", },
125+
body: JSON.stringify(new_collection)
126+
})
127+
.then(response => response.json())
128+
.then(data => {
129+
//console.log(data);
130+
this.collections.push(fixCollection(data));
131+
this.selectCollection(new_id.toString());
132+
})
133+
.catch(err => alert(err))
134+
.finally(() => { console.log('collection added'); });
135+
},
136+
showDetails(uri) {
137+
/// Show number of triples in collection
138+
getNumTriples(uri, num => {
139+
this.collectionInfo = `The collection <${uri}> contains ${num} triples.`;
140+
$('#collectionInfoMDialog').modal('show');
141+
});
142+
},
143+
deleteCollData(id) {
144+
if (this.displayCollection == null) {
145+
alert('No collection selected for data deletion.');
146+
return;
147+
}
148+
if (!confirm(`Are you sure you want to delete all data for collection ${id}? This action cannot be undone.`)) {
149+
return;
150+
}
151+
showProgess(true);
152+
var req = { method: 'POST', headers: { 'Accept': 'application/json, text/plain, */*' } }
153+
fetch(collection_url(`${id}/remove`), req)
154+
.then(response => response.json())
155+
.then(json => console.log(json))
156+
.catch(err => console.error(err))
157+
.finally(() => {
158+
showProgess(false);
159+
alert('Data deletion completed.');
160+
});
161+
},
162+
saveRDFUrl(event) {
163+
// Saves x3ml event data
164+
this.rdfFiles = event.target.files;
165+
},
166+
uploadRDF() {
167+
/// Upload RDF data from selected file
168+
if (this.rdfFiles.length ==0) {
169+
alert('No RDF file selected for upload.');
170+
}
171+
else {
172+
const file0 = this.rdfFiles[0]
173+
const textPromise = file0.text();
174+
textPromise.then((data) => {
175+
let suffix = file0.name.split('.').pop().toLowerCase();
176+
postCollectionData(this.displayCollection.uri, data, suffix);
177+
this.rdfFiles = [];
178+
});
179+
}
180+
},
181+
putCollection(collection) {
182+
/// Update collection metadata
183+
if (collection == null) {
184+
alert('No collection selected for update.');
185+
return;
186+
}
187+
const headers = { "Content-Type": "application/json", }
188+
fetch(collection_url(collection.id), {
189+
method: 'PUT',
190+
headers: headers,
191+
body: JSON.stringify(collection)
192+
})
193+
.then(response => response.json())
194+
.then(data => { console.log(data); })
195+
.catch(err => alert(err))
196+
.finally(() => {
197+
this.getCollection(collection.id);
198+
alert('collection changed');
199+
});
200+
},
201+
getCollection(id) {
202+
/// Retrieve collection metadata
203+
if (!id) {
204+
alert('No collection selected for retrieval.');
205+
return;
206+
}
207+
const headers = { "Accept": "application/json, text/plain, */*", }
208+
fetch(collection_url(id), { method: 'GET', headers: headers })
209+
.then(response => response.json())
210+
.then(data => { this.displayCollection = fixCollection(data); })
211+
.catch(err => alert(err))
212+
.finally(() => { console.log('collection fetched'); });
213+
},
214+
addType() {
215+
/// Add a new type to the collection
216+
if (this.displayCollection == null) {
217+
alert('No collection selected for adding type.');
218+
return;
219+
}
220+
this.displayCollection.type.push('x:y');
221+
},
222+
delType() {
223+
/// Remove a type from the collection
224+
if (this.displayCollection == null) {
225+
alert('No collection selected for deleting type.');
226+
return;
227+
}
228+
let N = this.displayCollection.type.length
229+
let id = prompt('Enter the index of the type to delete:', N);
230+
id = parseInt(id) - 1;
231+
if (!isNaN(id) && id >= 0 || id < N) {
232+
this.displayCollection.type.splice(id, 1);
233+
}
234+
},
235+
},
236+
mounted() {
237+
document.onreadystatechange = () => {
238+
if (document.readyState == "complete") {
239+
getCollections();
240+
}
241+
}
242+
},
243+
created() {
244+
console.log('created');
245+
}
246+
}
247+
}

0 commit comments

Comments
 (0)