Cleanup, comments, able to filter on backup files

Author: glormph

src/backend/analysis/tests.py

@@ -770,7 +770,7 @@ def test_new_analysis_and_run_and_purge(self):
             'analysisserver_id': self.anaserver.pk,
             }
         resp = self.cl.post(self.url, content_type='application/json', data=postdata)
-        timestamp = datetime.strftime(datetime.now(), '%Y%m%d_')
+
         self.assertEqual(resp.status_code, 200)
         ana = am.Analysis.objects.last()
         self.assertEqual(ana.analysissampletable.samples, {'hello': 'yes'})
@@ -797,6 +797,7 @@ def test_new_analysis_and_run_and_purge(self):
                 data={'analysis_id': resp.json()['analysis_id']})
         self.user.is_superuser = True
         self.user.save()
+        # Make sure rename lib file waits until analysis is finished
         renameresp = self.cl.post('/files/rename/', content_type='application/json',
                 data={'sf_id': self.sflib.pk, 'newname': 'testnewname'})
         self.user.is_superuser = False
@@ -814,7 +815,7 @@ def test_new_analysis_and_run_and_purge(self):
         self.assertTrue(usm_q.exists())
         self.assertTrue(hm.AnalysisMessage.objects.filter(analysis=ana, msg=usm_q.get().pk,
             msgtype=hm.AnalysisMsgTypes.COMPLETED).exists())
-        self.run_job() # rsync the results
+        self.run_job() # rsync the results, rename job
         self.assertTrue(renamejobs.filter(state=jj.Jobstates.PROCESSING).exists())
         reports = rm.StoredFileLoc.objects.filter(sfile__filename='report.html', purged=False)
         self.assertEqual(reports.count(), 3) # analysisruns, web, analysis storage

src/backend/datasets/models.py

@@ -113,6 +113,7 @@ class Dataset(models.Model):
     datatype = models.ForeignKey(Datatype, on_delete=models.CASCADE)
     securityclass = models.IntegerField(choices=DataSecurityClass.choices)
     deleted = models.BooleanField(default=False) # for UI only, indicate deleted from active storage
+    # deprecate purged
     purged = models.BooleanField(default=False) # for UI only, indicate permanent deleted from cold storage too
     locked = models.BooleanField(default=False)
 

src/backend/datasets/tasks.py

@@ -10,7 +10,6 @@
 from kantele import settings
 from jobs.post import get_session_cookies, update_db, taskfail_update_db
 from analysis.tasks import create_runname_dirname, prepare_nextflow_run, run_nextflow, register_mzmlfile, process_error_from_nf_log, copy_stage_files
-from rawstatus.tasks import calc_md5, delete_empty_dir
 
 # Updating stuff in tasks happens over the API, assume no DB is touched. This
 # avoids setting up auth for DB
@@ -38,12 +37,11 @@ def run_convert_mzml_nf(self, run, params, raws, ftype_name, nf_version, stagesc
     try:
         run_outdir = run_nextflow(run, params, run['dsspath'], gitwfdir, nf_version, scratchdir)
     except subprocess.CalledProcessError as e:
-        print(e)
         errmsg = process_error_from_nf_log(os.path.join(gitwfdir, '.nextflow.log'))
         taskfail_update_db(self.request.id, errmsg)
         raise RuntimeError('Error occurred converting mzML files: '
                            f'{basedir}\n\nERROR MESSAGE:\n{errmsg}')
-    # Technically we can dump straight to infile paths
+    # mzMLs go straight to infile paths
     token = False
     transfer_url = urljoin(settings.KANTELEHOST, reverse('jobs:updatestorage'))
     reg_session, reg_headers = get_session_cookies()

src/backend/datasets/views.py

@@ -619,7 +619,8 @@ def update_storage_shares(share_ids, project, experiment, dset, dtype, prefrac,
 
         elif pre_alldss.exists():
             # there is a live dss somewhere, get it onto an rsync source first
-            # This is VERY similar to what we do in home/views for mzml, except the local storage
+            # This is VERY similar to what we do in home/views around L1050 and L1180
+            # for mzml, except the local storage
             src_sfloc_ids = [x['pk'] for x in existing_sfl.filter(
                 servershare=pre_alldss.first().storageshare,
                     sfile__rawfile__datasetrawfile__dataset=dset).values('pk')]
@@ -1151,6 +1152,7 @@ def purge_project(request):
 
 
 def get_dset_storestate(dset_id, ds_deleted):
+    # For archiving, reactivating, and showing on homepage
     '''Count SFLoc associated with a dataset (created when adding/removing), in different 
     servershares and in backup - return a state for the dataset (currently a string)'''
     dsf_c = filemodels.StoredFile.objects.exclude(mzmlfile__isnull=False).filter(
@@ -1298,7 +1300,9 @@ def reactivate_dataset(dset, share_ids, project, experiment, dtype, prefrac, hrr
             sfls_inbox.update(path=buppath, active=True)
 
             sflids = [x['pk'] for x in sfls_inbox.values('pk')]
+            # Queue restore job from backup
             create_job('reactivate_dataset', dss_id=bupdss.pk, sfloc_ids=sflids)
+            # Now move from restored to other areas
             if upd_stor_err := update_storage_shares(share_ids, project, experiment, dset,
                     dtype, prefrac, hrrange, user_id):
                 msg = json.loads(upd_stor_err.content)['error']
@@ -1757,7 +1761,6 @@ def save_or_update_files(data, user_id):
             return {'error': 'Cannot find some of the files asked to remove on storage. '
                     'This should not happen. Contact admin' }, 403
 
-
     # Errors checked, now store DB records and queue move jobs
     if added_fnids:
         # First restore from backup

src/backend/home/views.py

@@ -183,6 +183,10 @@ def find_files(userquery):
                 query &= Q(deleted=True)
             case ['deleted', 'false']:
                 query &= Q(deleted=False)
+            case ['backup', 'false']:
+                query &= Q(pdcbackedupfile__success=False)
+            case ['backup', 'true']:
+                query &= Q(pdcbackedupfile__success=True)
             case ['type', ftype]:
                 match ftype:
                     case 'qc':
@@ -224,9 +228,10 @@ def getxbytes(bytes, op=50):
     if bytes is None:
         return '0B'
     if bytes >> op:
-        return '{}{}B'.format(bytes >> op, {0: '', 10: 'K', 20: 'M', 30: 'G', 40: 'T', 50: 'P'}[op])
+        unit = {0: '', 10: 'K', 20: 'M', 30: 'G', 40: 'T', 50: 'P'}[op]
+        return f'{bytes >> op}{unit}B'
     else:
-        return getxbytes(bytes, op-10)
+        return getxbytes(bytes, op - 10)
 
 
 def populate_files(dbfns):
@@ -1131,10 +1136,6 @@ def create_mzmls(request):
     return JsonResponse({})
 
 
-    
-
-
-
 @require_POST
 @login_required
 def refine_mzmls(request):

src/backend/jobs/jobs.py

@@ -125,7 +125,6 @@ def _get_extrafiles_to_rsync(self, **kwargs):
         return []
 
     def _select_rsync_server(self, srcshare_id, dstshare_id):
-        # Select file servers to rsync from/to - FIXME share code!
         '''Shares should already have been confirmed to be active?
         Or can we do that here?
         '''

src/backend/jobs/management/commands/runjobs.py

@@ -135,14 +135,15 @@ def run_ready_jobs(job_fn_map, job_ds_map, active_jobs):
                 job.state = Jobstates.PROCESSING
                 if errmsg := jwrapper.check_error_on_running(**job.kwargs):
                     jwrapper.set_error(job, errmsg=errmsg)
+                    print(f'Error found in check: "{errmsg}" --- not executing this job')
                 else:
                     try:
                         jwrapper.run(**job.kwargs)
                     except RuntimeError as e:
-                        print('Error occurred, trying again automatically in next round')
+                        print(f'Error occurred: {e}, not executing this job')
                         jwrapper.set_error(job, errmsg=str(e))
                     except Exception as e:
-                        print(f'Error occurred: {e} --- not executing this job')
+                        print(f'Uncaught error occurred: {e} --- not executing this job')
                         jwrapper.set_error(job, errmsg=str(e))
                     else:
                         # PROCESSING state update saved here:

src/backend/jobs/views.py

@@ -307,6 +307,7 @@ def restored_archive_file(request):
 
 @require_POST
 def analysis_run_done(request):
+    '''Called when successfully finished analysis'''
     data = json.loads(request.body.decode('utf-8'))
     if ('client_id' not in data or
             data['client_id'] not in settings.CLIENT_APIKEYS):

src/backend/kantele/settings.py

@@ -47,9 +47,6 @@
 RSYNC_SSHKEY = os.environ.get('RSYNC_SSHKEY')
 RSYNC_SSHPORT = os.environ.get('RSYNC_SSHPORT')
 
-SECONDARY_STORAGE_RSYNC_USER = os.environ.get('SECONDARY_STORAGE_RSYNC_USER')
-SECONDARY_STORAGE_RSYNC_KEY = os.environ.get('SECONDARY_STORAGE_RSYNC_KEY')
-
 UPLOAD_URL = 'uploads'
 
 

src/backend/kantele/tests.py

@@ -3,6 +3,7 @@
 import shutil
 from time import sleep
 from datetime import timedelta
+from unittest import TestCase
 
 from django.contrib.auth.models import User
 from django.test import TestCase, LiveServerTestCase, Client
@@ -32,6 +33,7 @@ def post_json(self, data):
 
     def setUp(self):
         # Clean directory containing storage servers
+        print('--------', TestCase.id(self))
         self.rootdir = '/storage'
         for dirname in os.listdir(self.rootdir):
             if os.path.isdir(os.path.join(self.rootdir, dirname)):