Skip to content

Commit 1377163

Browse files
alexandermichelsalexandermichels
committed
adding docs for using our API
1 parent 921f6f0 commit 1377163

File tree

1 file changed

+360
-0
lines changed

1 file changed

+360
-0
lines changed

‎docs/UsingAPI.md‎

Lines changed: 360 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,360 @@
1+
# Using the CyberGIS-Compute API
2+
3+
Basic demo: https://github.com/cybergis/cybergis-compute-web
4+
5+
Demo web app: https://cybergis.github.io/cybergis-compute-web/
6+
7+
CyberGIS-Compute Server Base URL: https://cgjobsup-test.cigi.illinois.edu/v2/
8+
9+
**Note:** We will ask web apps to move to our production server (https://cgjobsup.cigi.illinois.edu/v2/) once the required changes have been deployed there.
10+
11+
Swagger Docs: https://cgjobsup-test.cigi.illinois.edu/v2/api-docs/
12+
13+
## Authentication
14+
15+
Authentication uses tokens that you can request from the CyberGIS Center.
16+
17+
To verify that your token is working and/or recieve your CyberGIS-Compute username, you can use the `/user` route:
18+
19+
```
20+
await fetch('https://cgjobsup-test.cigi.illinois.edu/v2/user',
21+
{
22+
method: "POST",
23+
headers: {
24+
'Content-Type': 'application/json',
25+
},
26+
body: JSON.stringify({"jupyterhubApiToken": "YOUR_TOKEN_GOES_HERE"})
27+
})
28+
```
29+
30+
## Submitting a Job
31+
32+
**Note:** Data upload through the browser is still being developed.
33+
34+
There are a few steps required to submit a job.
35+
36+
1. POST /job - to create the job object in the backend and get an ID
37+
2. PUT /job/:id - to set the data for the Job object
38+
3. POST /job/:id/submit - to tell the backend the job should be submitted
39+
40+
It is likely that you want to call all three routes in succession when an end-user wants to submit a job.
41+
42+
#### POST /job
43+
44+
This creates the Job object and gives you can ID for interacting with the job. Example:
45+
46+
```
47+
await fetch('https://cgjobsup-test.cigi.illinois.edu/v2/job',
48+
{
49+
method: "POST",
50+
headers: {
51+
'Content-Type': 'application/json',
52+
},
53+
body: JSON.stringify(
54+
{
55+
'maintainer': "community_contribution",
56+
"jupyterhubApiToken": "YOUR_TOKEN_GOES_HERE",
57+
"hpc": selectedHpc ? selectedHpc : "keeling_community"
58+
}
59+
)
60+
})
61+
```
62+
63+
#### PUT /job/:id
64+
65+
This sets the details of the job. Example:
66+
67+
```
68+
await fetch('https://cgjobsup-test.cigi.illinois.edu/v2/job/' + jobID,
69+
{
70+
method: "PUT",
71+
headers: {
72+
'Content-Type': 'application/json',
73+
},
74+
body: JSON.stringify(
75+
{
76+
"jupyterhubApiToken": "YOUR_TOKEN_GOES_HERE",
77+
"localExecutableFolder": {"type": "git", "gitId": "herop_spatial_access"},
78+
"slurm": {"time": "30:00", "memory": "16GB"},
79+
"param": {
80+
"mobility_mode": "DRIVING",
81+
"population_type": "ZIP",
82+
"max_travel_time": "30",
83+
"access_measure": "ALL",
84+
"supply_filename": "supply/ContinentalHospitals.shp",
85+
"supply_capacity": "BEDS",
86+
"supply_latlon_or_id": "ID",
87+
"supply_id": "ZIP",
88+
"supply_lat": "",
89+
"supply_lon": ""
90+
},
91+
}
92+
)
93+
})
94+
```
95+
96+
#### POST /job/:id/submit
97+
98+
Lastly, you want to submit the job. Example:
99+
100+
```
101+
await fetch('https://cgjobsup-test.cigi.illinois.edu/v2/job/' + jobID + '/submit',
102+
{
103+
method: "POST",
104+
headers: {
105+
'Content-Type': 'application/json'
106+
},
107+
body: JSON.stringify({"jupyterhubApiToken": "YOUR_TOKEN_GOES_HERE"})
108+
})
109+
```
110+
111+
## Monitoring the Job
112+
113+
You can get a job's status using the POST /job/:id route. Example:
114+
115+
```
116+
await fetch('https://cgjobsup-test.cigi.illinois.edu/v2/job/' + jobID,
117+
{
118+
method: "POST",
119+
headers: {
120+
'Content-Type': 'application/json'
121+
},
122+
body: JSON.stringify({"jupyterhubApiToken": "YOUR_TOKEN_GOES_HERE"})
123+
})
124+
```
125+
126+
This route returns a large JSON dictionary with a variety of information about the job. You will know the job is complete when the "finishedAt" field is no longer null.
127+
128+
```
129+
{
130+
"id": "1745852151TqPs8",
131+
"userId": "alexandermichels@cybergisx.cigi.illinois.edu",
132+
"name": null,
133+
"maintainer": "community_contribution",
134+
"hpc": "keeling_community",
135+
"remoteExecutableFolder": {
136+
"id": "1745852155PJYxn",
137+
"name": null,
138+
"hpc": "keeling_community",
139+
"hpcPath": "/data/keeling/a/cigi-gisolve/scratch/1745852155PJYxn",
140+
"globusPath": "/1745852155PJYxn",
141+
"userId": "alexandermichels@cybergisx.cigi.illinois.edu",
142+
"isWritable": false,
143+
"createdAt": "2025-04-28T14:55:57.540Z",
144+
"updatedAt": null,
145+
"deletedAt": null
146+
},
147+
"remoteDataFolder": null,
148+
"remoteResultFolder": {
149+
"id": "1745852158f8NuN",
150+
"name": null,
151+
"hpc": "keeling_community",
152+
"hpcPath": "/data/keeling/a/cigi-gisolve/scratch/1745852158f8NuN",
153+
"globusPath": "/1745852158f8NuN",
154+
"userId": "alexandermichels@cybergisx.cigi.illinois.edu",
155+
"isWritable": false,
156+
"createdAt": "2025-04-28T14:55:58.154Z",
157+
"updatedAt": null,
158+
"deletedAt": null
159+
},
160+
"localExecutableFolder": {
161+
"type": "git",
162+
"gitId": "herop_spatial_access"
163+
},
164+
"localDataFolder": null,
165+
"param": {
166+
"mobility_mode": "DRIVING",
167+
"population_type": "ZIP",
168+
"max_travel_time": "30",
169+
"access_measure": "ALL",
170+
"supply_filename": "supply/ContinentalHospitals.shp",
171+
"supply_capacity": "BEDS",
172+
"supply_latlon_or_id": "ID",
173+
"supply_id": "ZIP",
174+
"supply_lat": "",
175+
"supply_lon": ""
176+
},
177+
"env": {},
178+
"slurm": {
179+
"time": "30:00",
180+
"memory": "16GB"
181+
},
182+
"slurmId": "9452091",
183+
"credentialId": null,
184+
"events": [
185+
{
186+
"id": 15475,
187+
"jobId": "1745852151TqPs8",
188+
"type": "JOB_QUEUED",
189+
"message": "job [1745852151TqPs8] is queued, waiting for registration",
190+
"createdAt": "2025-04-28T14:55:51.666Z",
191+
"updatedAt": null,
192+
"deletedAt": null
193+
},
194+
{
195+
"id": 15476,
196+
"jobId": "1745852151TqPs8",
197+
"type": "JOB_REGISTERED",
198+
"message": "job [1745852151TqPs8] is registered with the supervisor, waiting for initialization",
199+
"createdAt": "2025-04-28T14:55:53.472Z",
200+
"updatedAt": null,
201+
"deletedAt": null
202+
},
203+
{
204+
"id": 15477,
205+
"jobId": "1745852151TqPs8",
206+
"type": "SLURM_UPLOAD_EXECUTABLE",
207+
"message": "uploading executable folder",
208+
"createdAt": "2025-04-28T14:56:00.078Z",
209+
"updatedAt": null,
210+
"deletedAt": null
211+
},
212+
{
213+
"id": 15478,
214+
"jobId": "1745852151TqPs8",
215+
"type": "SSH_SCP_UPLOAD",
216+
"message": "put file from /job_supervisor/data/root/herop_spatial_access.zip to /data/keeling/a/cigi-gisolve/scratch/cache/herop_spatial_access.zip",
217+
"createdAt": "2025-04-28T14:56:00.086Z",
218+
"updatedAt": null,
219+
"deletedAt": null
220+
},
221+
{
222+
"id": 15479,
223+
"jobId": "1745852151TqPs8",
224+
"type": "SSH_UNZIP",
225+
"message": "unzipping /data/keeling/a/cigi-gisolve/scratch/cache/herop_spatial_access.zip to /data/keeling/a/cigi-gisolve/scratch/1745852155PJYxn",
226+
"createdAt": "2025-04-28T14:56:00.094Z",
227+
"updatedAt": null,
228+
"deletedAt": null
229+
},
230+
{
231+
"id": 15480,
232+
"jobId": "1745852151TqPs8",
233+
"type": "SLURM_CREATE_RESULT",
234+
"message": "create result folder",
235+
"createdAt": "2025-04-28T14:56:00.103Z",
236+
"updatedAt": null,
237+
"deletedAt": null
238+
},
239+
{
240+
"id": 15481,
241+
"jobId": "1745852151TqPs8",
242+
"type": "SSH_MKDIR",
243+
"message": "removing /data/keeling/a/cigi-gisolve/scratch/1745852158f8NuN/slurm_log",
244+
"createdAt": "2025-04-28T14:56:00.112Z",
245+
"updatedAt": null,
246+
"deletedAt": null
247+
},
248+
{
249+
"id": 15482,
250+
"jobId": "1745852151TqPs8",
251+
"type": "SSH_SCP_UPLOAD",
252+
"message": "put file from /job_supervisor/data/tmp/tmp-no2vdssoo7 to /data/keeling/a/cigi-gisolve/scratch/1745852155PJYxn/job.sbatch",
253+
"createdAt": "2025-04-28T14:56:00.122Z",
254+
"updatedAt": null,
255+
"deletedAt": null
256+
},
257+
{
258+
"id": 15483,
259+
"jobId": "1745852151TqPs8",
260+
"type": "SSH_CREATE_FILE",
261+
"message": "create file to /data/keeling/a/cigi-gisolve/scratch/1745852155PJYxn/job.json",
262+
"createdAt": "2025-04-28T14:56:00.129Z",
263+
"updatedAt": null,
264+
"deletedAt": null
265+
},
266+
{
267+
"id": 15484,
268+
"jobId": "1745852151TqPs8",
269+
"type": "SSH_SCP_UPLOAD",
270+
"message": "put file from /job_supervisor/data/tmp/tmp-oiwpt78e5j to /data/keeling/a/cigi-gisolve/scratch/1745852155PJYxn/job.json",
271+
"createdAt": "2025-04-28T14:56:00.136Z",
272+
"updatedAt": null,
273+
"deletedAt": null
274+
},
275+
{
276+
"id": 15485,
277+
"jobId": "1745852151TqPs8",
278+
"type": "SLURM_SUBMIT",
279+
"message": "submitting slurm job",
280+
"createdAt": "2025-04-28T14:56:00.144Z",
281+
"updatedAt": null,
282+
"deletedAt": null
283+
},
284+
{
285+
"id": 15486,
286+
"jobId": "1745852151TqPs8",
287+
"type": "SLURM_SUBMIT_SUCCESS",
288+
"message": "slurm job submitted with slurm job id 9452091",
289+
"createdAt": "2025-04-28T14:56:00.152Z",
290+
"updatedAt": null,
291+
"deletedAt": null
292+
},
293+
{
294+
"id": 15487,
295+
"jobId": "1745852151TqPs8",
296+
"type": "JOB_INIT",
297+
"message": "job [1745852151TqPs8] is initialized, waiting for job completion",
298+
"createdAt": "2025-04-28T14:56:00.164Z",
299+
"updatedAt": null,
300+
"deletedAt": null
301+
}
302+
],
303+
"logs": [],
304+
"createdAt": "2025-04-28T14:55:51.493Z",
305+
"updatedAt": null,
306+
"deletedAt": null,
307+
"initializedAt": "2025-04-28T14:56:00.157Z",
308+
"finishedAt": null,
309+
"queuedAt": "2025-04-28T14:55:51.675Z",
310+
"isFailed": false,
311+
"nodes": null,
312+
"cpus": null,
313+
"cpuTime": null,
314+
"memory": null,
315+
"memoryUsage": null,
316+
"walltime": null
317+
}
318+
```
319+
320+
## Downloading Job Results
321+
322+
You can download the results of a job through the browser using the POST /folder/:folderID/download/browser route. The body requires:
323+
324+
* "jupyterhubApiToken": Authentication token
325+
* "jobId": the ID of the job
326+
* "folderId": the ID of the folder you would like to download. You can get this field from the job status dictionary using `jobStatusResponse['remoteResultFolder']['id']` (NOTE: jobStatusResponse here is the status request response).
327+
328+
Example:
329+
330+
```
331+
await fetch('https://cgjobsup-test.cigi.illinois.edu/v2/folder/' + downloadID + '/download/browser',
332+
{
333+
method: "POST",
334+
headers: {
335+
'Content-Type': 'application/json',
336+
},
337+
body: JSON.stringify(
338+
{
339+
"jupyterhubApiToken": "YOUR_TOKEN_GOES_HERE",
340+
"jobId": jobID,
341+
"folderId": downloadID
342+
}
343+
)
344+
})
345+
.then(async function (response) {
346+
if (response.status !== 200) {
347+
console.log(
348+
'Looks like there was a problem. Status Code: ' + response.status
349+
);
350+
return;
351+
}
352+
const blob = await response.blob();
353+
const objectUrl = URL.createObjectURL(blob);
354+
const link = document.createElement('a');
355+
link.href = objectUrl;
356+
link.download = 'result.zip';
357+
link.click();
358+
URL.revokeObjectURL(objectUrl);
359+
})
360+
```

0 commit comments

Comments
 (0)