add instructions and fix

Author: osallou

bioconda-backup/README.md

@@ -29,9 +29,40 @@ Support full scan update or update from last repo check
 
 node index.js --help
 
+## CI
+
+github CI will execute at regular interval. Script will check in workdir (see config.yml) for git repo clone.
+
+* if not present, will clone repo and scan for all containers
+* if present, will look *record* last commit, pull data, and look at modified files (so containers) to check only related containers (if they need a backup)
+
+GitHub stops jobs after 24 hours, which can occur for very large updates. In this case, script is killed and a lock file will be left, preventing further cron jobs to run.
+Lock file is to precent running multiple jobs at the same time. If this occurs (**wokrdir**/sync.lock already present), simply delete the lock file and run
+manually the failed job (better in background...):
+
+    # workdir = /mnt/data/sync
+    # look at json file related to date failure
+    ls -l /mnt/data/sync
+    drwxr-xr-x   7 ubuntu ubuntu    4096 Jul 22 23:09 bioconda
+    drwxr-xr-x 902 ubuntu ubuntu   36864 Jul 23 00:40 biocontainers
+    -rw-r--r--   1 ubuntu ubuntu 1103453 Jul 20 23:49 biocontainers_1689894554195.json
+    -rw-rw-r--   1 ubuntu ubuntu       0 Jul 23 07:10 sync.lock
+    # example
+    node index.js --backup --aws --conda --file /mnt/data/sync/biocontainers_1689894554195.json
+
+json files are created at container lookup step, and deleted on success. In case of job failure, json file will be present and ease replay.
+final solution is to delete **workdir**/bioconda|biocontainer and run without the --updated option to check for **all** containers (very long job....)
 
 ## example
 
+### backup conda containers to registry AND aws
+
 backup conda (quay.io) containers to aws since last run
 
 node index.js --aws --conda --updated
+
+### backup github dockerfile containers to registry AND aws
+
+backup conda (quay.io) containers to aws since last run
+
+node index.js --aws --docker --updated

bioconda-backup/index.js

@@ -585,19 +585,31 @@ async function getContainers(scan_options) {
   let containers = []
   let dockerfiles = await scan('biocontainers', scan_options)
   let docker_containers = []
+  let docker_managed = {}
   for(let i=0;i<dockerfiles.length;i++){
     let elts = dockerfiles[i].split('/');
+    if(docker_managed[elts[elts.length-3]] !== undefined) {
+      continue
+    }
     docker_containers.push({'name': elts[elts.length-3], 'tags': [], quay: false})
+    docker_managed[elts[elts.length-3]] = true
   }
+  docker_managed = null
   let data = await dockerhub(docker_containers, scan_options)
   containers = containers.concat(data)
 
   let condafiles = await scan('bioconda', scan_options)
   let conda_containers = []
+  let conda_managed = {}
   for(let i=0;i<condafiles.length;i++){
     let elts = condafiles[i].split('/');
+    if(conda_managed[elts[elts.length-2]] !== undefined) {
+      continue
+    }
     conda_containers.push({'name': elts[elts.length-2], 'tags': [], quay: true});
+    conda_managed[elts[elts.length-2]] = true
   }
+  conda_managed = null
   data = await quayio(conda_containers, scan_options)
   containers = containers.concat(data)