Merge branch 'master' into production

Author: jay-m-dev

.env

@@ -7,5 +7,5 @@
 #   Leave this set to 0 on the GitHub repo so the unit and 
 #      integration tests do not need to have wheels (until we 
 #      find a convenient way to use wheels on GitHub)
-TAG=0.20
+TAG=0.21
 USE_WHEELS=1

.github/workflows/add-to-project.yml

@@ -0,0 +1,15 @@
+name: Add issue to project
+on:
+  issues:
+    types: [opened]
+  pull_request:
+    types: [opened]
+jobs:
+  add-to-project:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Add issue to project
+        uses: actions/add-to-project@v0.5.0
+        with:
+          github_token: ${{ secrets.PENNAI_TOKEN }}
+          project_url: https://github.com/orgs/EpistasisLab/projects/9

docker_logs.txt

@@ -54,6 +54,9 @@ db.bind("batches");
 db.bind("users");
 db.bind("datasets");
 db.bind("settings");
+db.bind("chats");
+db.bind("chatlogs");
+db.bind("executions");
 
 // Promisify all methods
 Object.keys(mongoskin).forEach((key) => {

lab/db.js

@@ -54,6 +54,7 @@ const assert = require("assert");
 const openaiRouter = require('./routes/openai');
 const chatapiRouter = require('./routes/chatapi');
 const execapiRouter = require('./routes/execapi');
+const { deleteFilesFromGridstore } = require('./labutils');
 
 /***************
 * Enums
@@ -601,6 +602,7 @@ app.get("/api/v1/:collection/:id", (req, res, next) => {
 
 // Update existing entry
 app.put("/api/v1/:collection/:id", jsonParser, (req, res, next) => {
+
     delete req.body._id; // Delete ID (will not update otherwise)
     req.collection.updateByIdAsync(req.params.id, {
             $set: req.body
@@ -626,6 +628,99 @@ app.put("/api/v1/:collection/:id", jsonParser, (req, res, next) => {
         });
 });
 
+app.delete('/api/v1/datasets/:id', async (req, res, next) => {
+    const result = {};
+    let files = [];
+    let query = '';
+    try {
+        const dataset_id = db.toObjectID(req.params.id);
+        let dataset = await db.datasets.findByIdAsync(dataset_id);
+        
+        if (dataset == null) {
+            return res.status(404).send({ message: 'dataset ' + req.params.id + ' not found'});
+        }
+
+        // if (dataset.ai && (dataset.ai.status == recommenderStatus.RUNNING || dataset.ai.status == recommenderStatus.INITIALIZING)) {
+        if (dataset.ai && ['on', 'requested'].some(status => dataset.ai.status === status)) {
+            console.log('cannot delete dataset, AI is enabled');
+            return res.status(409).send({message: 'cannot delete dataset, AI is enabled'});
+        }
+        
+        const dataset_file_id = db.toObjectID(dataset.files[0]._id);
+        files.push(...dataset.files);
+        
+        // experiments
+        query = { $or: [
+            {"_dataset_id": {"$eq": dataset_id}},
+            {"_dataset_file_id": {"$eq": dataset_file_id}},
+        ]}
+
+        let experiments = await db.experiments.find(query).toArrayAsync();
+        console.log(experiments);
+        let runningExp = experiments.find(exp => ['running', 'pending', 'suggested'].some(status => exp._status === status));
+
+        if (runningExp) {
+            console.log('cannot delete dataset, experiments running');
+            return res.status(409).send({message: 'cannot delete dataset, experiments running'});
+        }
+
+        let experimentIds = []; // maybe I don't need this one.
+        experiments.forEach(exp => {
+            experimentIds.push(exp._id);
+            files.push(...exp.files);
+        })
+        
+        // chats
+        query = { $or: [
+            {"_dataset_id": {"$eq": dataset_id}},
+            {"_experiment_id": {"$in": experimentIds}}
+        ]}
+        let chats = await db.chats.find(query).toArrayAsync();
+        let chatIds = [];
+        let chatlogIds = [];
+        chats.forEach(chat => {
+            chatIds.push(chat._id);
+            chatlogIds.push(...chat.chatlogs);
+        })
+        
+        // executions
+        query = { $or: [
+            {"_dataset_id": {"$eq": dataset_id}},
+            {"_dataset_file_id": {"$eq": dataset_file_id}},
+            {"_experiment_id": {"$in": experimentIds}}
+        ]}
+        let executions = await db.executions.find(query).toArrayAsync();
+        executions.forEach(exec => {
+            files.push(...exec.files);
+        })
+
+        // *** DELETE ***
+        result.datasetCount = await db.datasets.removeByIdAsync(dataset_id);
+        if (experiments.length > 0) {
+            result.experimentCount = await db.experiments.removeAsync({'_id': { '$in': experimentIds }});
+            console.log('experiment count');
+        }
+        if (chatIds.length > 0) {
+            result.chatlogCount = (await db.chatlogs.removeAsync({'_id': { '$in': chatlogIds }}));
+            console.log('chatlogs deleted');
+            result.chatCount = (await db.chats.removeAsync({'_id': { '$in': chatIds }}));
+            console.log('chats deleted');
+        }
+        result.fileCount = await deleteFilesFromGridstore(files);
+
+        // temp values
+        // result.datasets = dataset;
+        // result.experiments = experiments;
+        // result.chats = chats;
+        // result.executions = executions;
+        // result.files = files;
+
+        res.send(result);
+    } catch (err) {
+        next(err);
+    }
+});
+
 // Delete existing entry
 app.delete("/api/v1/:collection/:id", (req, res, next) => {
     req.collection.removeByIdAsync(req.params.id)
@@ -643,7 +738,6 @@ app.delete("/api/v1/:collection/:id", (req, res, next) => {
 });
 
 
-
 // Experiment page
 app.get("/api/v1/experiments/:id", (req, res, next) => {
     db.experiments.findByIdAsync(req.params.id)

lab/lab.js

@@ -0,0 +1,27 @@
+const db = require("./db").db;
+
+async function deleteFilesFromGridstore(files) {
+  try {
+    let filesP = [];
+    let filesDeleted = 0;
+
+    for (let i = 0; i < files.length; i++) {
+      let gfs = new db.GridStore(db, files[i]._id, 'w', {
+        promiseLibrary: Promise
+      });
+      filesP.push(gfs.unlinkAsync().then(() => {
+        filesDeleted++;
+      }));
+    }
+
+    await Promise.all(filesP);
+    return filesDeleted;
+  } catch (err) {
+    console.log(err);
+    throw err;
+  }
+}
+
+module.exports = {
+  deleteFilesFromGridstore
+}

lab/labutils.js

@@ -120,103 +120,6 @@ router.post('/executions', async (req, res, next) => {
 });
 
 
-router.post('/executions_old', async (req, res, next) => {
-    if (req.body.src_code == null) {
-        return res.status(400).json({ message: 'No src_code provided' });
-    }
-
-    // This should not be needed in the code run. The client should take the
-    // execution_id returned by this enpoint and write it to the next chatlog.
-    // // this will be the chatlog_id of the chatlog that is requesting this run.
-    // // the next chatlog will be the one with the results of this run. The client
-    // // will need to save the execution_id returned by this endpoint in the next
-    // // chatlog.
-    // if (req.body.chatlog_id != null) {
-    //     return res.status(400).json({ message: 'no chatlog_id provided' });
-    // }
-
-    // create a new execution
-    let execution = new Execution({
-        src_code: req.body.src_code,
-        status: 'submitted',
-        result: null,
-        files: []
-    });
-
-    if (req.body.dataset_file_id != null) {
-        execution._dataset_file_id = req.body.dataset_file_id;
-    } else if (req.body.dataset_id != null) {
-        execution._dataset_id = req.body.dataset_id;
-        let dataset = await getDatasetById(req.body.dataset_id);
-        if (dataset != null) {
-            execution._dataset_file_id = dataset.files[0]._id;
-        }
-    }
-
-    if (req.body.experiment_id != null) {
-        execution._experiment_id = req.body.experiment_id;
-    }
-
-    try {
-        const newExecution = await execution.save();
-        execution._id = newExecution._id;
-    } catch (err) {
-        return res.status(500).json({ message: err.message });
-    }
-
-    // make folder if not available yet:
-    // let tmppath = path.join(process.env.CODE_RUN_PATH, request.execution_id.toString());
-    let tmppath = path.join(process.env.CODE_RUN_PATH, execution._id.toString());
-    // make tmp folder if it is not available
-    if (!fs.existsSync(tmppath)) fs.mkdirSync(tmppath, { recursive: true });
-
-    // find machines that could handle the project
-    // this may need revision, submitting experiments checks the machine capacity
-    // but the capacity is tied to each algorithm. Not sure how to handle this yet.
-    let machines;
-    try {
-        machines = await Machine.find({}, { address: 1 });
-        if (machines.length == 0) {
-            return res.status(400).json({ message: 'No machines available' });
-        }
-        // call the machine api
-        let result = await fetch(machines[0].address + '/code/run', {
-            method: 'POST',
-            headers: {
-                'Content-Type': 'application/json'
-            },
-            body: JSON.stringify(execution)
-        });
-        result = await result.json();
-
-        // update the execution status
-        execution.status = result.exec_results.status;
-        execution.result = result.exec_results.result;
-
-        // add any generated files in tmppath to the execution.files array
-        const files = await uploadExecFiles(execution._id, tmppath);
-        execution.files = files;
-
-        const updatedExecution = await execution.save();
-
-        // delete the tmp folder
-        fs.rmdir(tmppath, { recursive: true }, (err) => {
-            if (err) {
-                console.error(err);
-            } else {
-                console.log(tmppath + ' folder deleted');
-            }
-        });
-
-        res.send(execution);
-    }
-    catch (err) {
-        console.error(err);
-        return res.status(500).json({ message: err.message });
-    }
-});
-
-
 router.post('/executions/install', async (req, res, next) => {
 
     if (req.body.command != 'install' && req.body.command != 'freeze') {