[bioconda][backup] add new scripts

Author: osallou

bioconda-backup/backup.py bioconda-backup-old/backup.pybioconda-backup/backup.py renamed to bioconda-backup-old/backup.py

@@ -0,0 +1,27 @@
+# containers backup
+
+Check for biocontainers and bioconda container updates to copy them
+to local backup registry and optionally to aws.
+Also add security scan
+
+Support full scan update or update from last repo check
+
+## setup
+
+* nodejs
+* aws cli and aws credentials set in ~/.aws
+
+    ls ~/.aws
+    config  credentials
+    
+    # config
+    [default]
+    region = us-east-1
+    # credentials
+    [default]
+    aws_access_key_id =
+    aws_secret_access_key =
+
+## run
+
+node index.js --help

bioconda-backup/backup.sh bioconda-backup-old/backup.shbioconda-backup/backup.sh renamed to bioconda-backup-old/backup.sh

@@ -0,0 +1,21 @@
+default:
+  mail:
+    to: '' # destination email
+    smtp: '' # smtp ip
+  s3:
+    endpoint: ''
+    access_key: ''
+    secret_access_key: ''
+    bucket: 'biocontainers2'
+
+  registry:
+    host: 'biocontainers.novalocal:5000'
+
+  anchore:
+    url: 'http://localhost:8228/v1'
+    user: 'admin'
+    password: ''
+  
+  aws: null # aws registry id
+
+  workdir: '/tmp'

bioconda-backup/env.sh.example bioconda-backup-old/env.sh.examplebioconda-backup/env.sh.example renamed to bioconda-backup-old/env.sh.example

@@ -0,0 +1,79 @@
+const fs = require('fs');
+const path = require('path');
+const Git = require("nodegit");
+
+const workdir = '/tmp';
+const biocontainers = 'https://github.com/BioContainers/containers.git';
+const bioconda = 'https://github.com/bioconda/bioconda-recipes.git';
+
+const getRepoFiles = function(dirPath, arrayOfFiles) {
+    let files = fs.readdirSync(dirPath)
+    arrayOfFiles = arrayOfFiles || []
+  
+    files.forEach(function(file) {
+        if (fs.statSync(dirPath + "/" + file).isDirectory()) {
+            arrayOfFiles = getRepoFiles(dirPath + "/" + file, arrayOfFiles)
+        } else {
+            let dirElt = dirPath.split('/');
+            if(file == 'Dockerfile' || (file == 'meta.yaml' && dirElt[dirElt.length-2] == 'recipes')) {
+            arrayOfFiles.push(path.join(dirPath, "/", file))
+            }
+        }
+    })
+  
+    return arrayOfFiles
+  }
+
+async function repoFiles(kind='biocontainers') {
+    let destDir = `${workdir}/${kind}`;
+    let lastCommit = null;
+    if (fs.existsSync(destDir)) {
+        let existingRepo = await Git.Repository.open(destDir);
+        let bc = await existingRepo.getBranchCommit('master');
+        lastCommit = bc.sha();
+        console.log('existing, last commit', lastCommit);
+        fs.rmSync(destDir, {recursive: true});
+    } 
+    let repoUrl = kind == 'biocontainers' ? biocontainers : bioconda;
+    await Git.Clone(repoUrl, destDir);
+    let repo = await Git.Repository.open(destDir);
+    let headc = await repo.getBranchCommit('master');
+    let headCommit = headc.sha();
+    console.log('last commit', lastCommit);
+    console.log('head commit', headCommit);
+    
+    let files = [];
+    if(lastCommit && lastCommit == headCommit) {
+        console.log('nothing to do');
+        return [];
+    }
+    if(lastCommit) {
+        console.log('get diff');
+        let cold = await repo.getCommit(lastCommit);
+        let coldTree = await cold.getTree();
+        let cnew = await repo.getCommit(headCommit);
+        let cnewTree = await cnew.getTree();
+        const diff = await cnewTree.diff(coldTree);
+        const patches = await diff.patches();
+        
+        for (const patch of patches) {
+            files.push(patch.newFile().path());
+        }
+        return files;
+
+    }
+    console.log('take all');
+    let allFiles = getRepoFiles(destDir);
+    for(let i=0;i<allFiles.length;i++) {
+        allFiles[i] = allFiles[i].replace(destDir + '/', '');
+    }
+    return allFiles;
+}
+
+repoFiles('biocontainers').then(files => {
+    console.log('files', files.length, files[0]);
+    process.exit(0);
+}).catch(err => {
+    console.error(err);
+    process.exit(1);
+})