Code Cleanup, README update

Author: m-murphy

README.md

@@ -1,8 +1,30 @@
 # MicroSPAT
 
-#To Do
-1. Add in controls for probabilistic allele identification at the project level
-2. refactor and pull control of a project into service
-3. refactor REST API into sockets based API
-4. Add in notification functionality
-  
+## Introduction
+MicroSPAT is a collection of tools for semi-automated analysis of raw capillary electrophoresis (CE) data output by the ABI 3730. MicroSPAT integrates several features including a plate view for quality checking, automated ladder identification, sample based association of FSA data to keep data organized in a logical manner, automated bin generation using a clustering algorithm, automated artifact estimator generation, automated quantification bias estimation, and automated genotyping of samples with the option of manual curation.
+
+## DEV WARNING
+Under Very Heavy Development -- Features will change, things WILL break
+
+## Installation
+Download the latest build zip, unzip, and navigate to the folder.  Using either pip or conda, install the required packages either through requirements.txt or using the conda environment manager and the environment.yml file.  If you plan to install through pip, I highly suggest using an environment manager, but this is beyond the scope of this tutorial. Once all packages are installed, execute the following
+
+```
+python manage.py initDB
+```
+
+This initializes the application with the HDROX400 Ladder
+
+## Running
+While in the MicroSPAT directory, execute the following
+
+```
+python manage.py runsockets
+```
+
+Then open Chrome (Only tested in Chrome to date, no guarantee for other browsers at this time) and navigate to:
+```
+http://localhost:5000
+```
+
+This will bring up MicroSPAT for the first time.  At this point, you can then start to populate the application with your data.

app/microspat/events.py

@@ -87,9 +87,6 @@ def update_peak_scanner(target, update_dict):
     :type update_dict: dict
     :type target: PeakScanner
     """
-    print update_dict
-    print type(update_dict)
-    print update_dict['scanning_method']
     if update_dict['scanning_method'] == 'relmax':
         target.scanning_method = 'relmax'
         target.argrelmax_window = update_dict['argrelmax_window']
@@ -258,7 +255,6 @@ def load_plate_map(plate_map_file, plate, create_samples_if_not_exist=False):
     plate.check_contamination()
     for project_id in new_channels.keys():
         project = Project.query.get(project_id)
-        print "Adding New Channels"
         project.add_channels(new_channels[project_id])
     db.session.commit()
     plate = Plate.query.get(plate.id)
@@ -298,7 +294,6 @@ def test_message(message=None):
 
 @socketio.on('client_test')
 def client_test(message=None):
-    print message
     emit('server_test', 'Success')
 
 
@@ -322,7 +317,6 @@ def get_or_post_genotyping_projects():
         return table_list_all(GenotypingProject)
     elif request.method == 'POST':
         project_params = json.loads(request.get_json())
-        print project_params
         try:
             project = GenotypingProject(**project_params)
             db.session.add(project)
@@ -473,7 +467,6 @@ def get_peak_data(id):
                 res["In Bin"] = bool(peak['bin_id'])
                 res["Relative Peak Height"] = peak['relative_peak_height']
                 if res["In Bin"]:
-                    print peak['bin_id']
                     res["Called Allele"] = str(peak['bin_id']) in bin_ids
                     res["Allele Label"] = str(peak['bin'])
                 res["Bleedthrough Ratio"] = peak['bleedthrough_ratio']
@@ -562,7 +555,6 @@ def get_or_post_bias_estimator_projects():
         return table_list_all(QuantificationBiasEstimatorProject)
     elif request.method == 'POST':
         project_params = json.loads(request.get_json())
-        print project_params
         try:
             project = QuantificationBiasEstimatorProject(**project_params)
             db.session.add(project)
@@ -715,7 +707,6 @@ def get_or_create_bin_estimators():
     elif request.method == 'POST':
         project_params = json.loads(request.get_json())
         try:
-            print "Adding a new project"
             project = BinEstimatorProject(**project_params)
             db.session.add(project)
             db.session.flush()
@@ -777,7 +768,6 @@ def create_or_update_bins(id, locus_id):
     # Remove deleted bins
     old_bin_ids = [_['id'] for _ in old_bins]
     for b in locus_bin_set.bins:
-        print b.id
         if b.id not in old_bin_ids:
             db.session.delete(b)
 
@@ -823,7 +813,6 @@ def batch_update_locus_parameters():
     proj_id = locus_params_update_dict['project_id']
     project = Project.query.get(proj_id)
     locus_parameters = ProjectLocusParams.query.filter(ProjectLocusParams.project_id == proj_id).all()
-    print project, locus_parameters
     if locus_parameters:
         try:
             updater = update_fns.get(locus_parameters[0].discriminator, update_locus_params)
@@ -858,7 +847,6 @@ def get_or_update_locus_parameters(id):
         return table_get_details(ProjectLocusParams, id)
     elif request.method == 'PUT':
         locus_params_update_dict = json.loads(request.get_json())
-        print type(locus_params_update_dict)
         locus_params = ProjectLocusParams.query.get(id)
         assert isinstance(locus_params, ProjectLocusParams)
         project = Project.query.get(locus_params.project_id)
@@ -868,7 +856,6 @@ def get_or_update_locus_parameters(id):
                 try:
                     locus_params = updater(locus_params, locus_params_update_dict)
                 except StaleParametersError as e:
-                    print "STALE PARAMETER ERROR"
                     return handle_error("{} is stale at locus {}, analyze that first!".format(e.project, e.locus))
                 db.session.flush()
                 send_notification('info', 'Beginning Analysis: {}'.format(locus_params.locus.label))
@@ -938,8 +925,6 @@ def get_or_post_locus_sets():
         return table_list_all(LocusSet)
     elif request.method == 'POST':
         request_json = request.get_json()
-        print request_json
-        print type(request_json)
         locus_set_params = json.loads(request_json['locus_set'])
         locus_ids = request_json['locus_ids']
         try:
@@ -979,7 +964,6 @@ def get_or_post_ladders():
     elif request.method == 'POST':
         ladder_params = json.loads(request.get_json())
         try:
-            print ladder_params
             if ladder_params.get('id', None):
                 l = Ladder.query.get(ladder_params['id'])
             else:
@@ -1066,7 +1050,6 @@ def save_plate():
                 files.append(os.path.join('./tmp', filename))
             ladder = Ladder.query.get(ladder_id)
             plate_zips = [open(_, 'rb') for _ in files]
-            print plate_zips
             try:
                 if os.name == 'nt':  # Currently no support for multiprocessing in windows at this time.
                     extracted_plates = load_plate_zips(plate_zips, ladder, parallel=False)
@@ -1243,7 +1226,6 @@ def get_project_sample_locus_annotations_by_sample(project_id, sample_id):
 def update_locus_annotations():
     annotations = map(json.loads, request.get_json())
     for annotation in annotations:
-        print annotation
         sample_annotation = SampleLocusAnnotation.query.get(annotation['id'])
         assert isinstance(sample_annotation, SampleLocusAnnotation)
         sample_annotation.alleles = annotation['alleles']
@@ -1259,7 +1241,6 @@ def get_controls():
     elif request.method == 'POST':
         try:
             ctrl_info = json.loads(request.get_json())
-            print ctrl_info
             ctrl = Control(barcode=ctrl_info['barcode'], bin_estimator_id=ctrl_info['bin_estimator_id'])
             for k in ctrl_info['alleles'].keys():
                 if ctrl_info['alleles'][k] == 'null':
@@ -1284,8 +1265,6 @@ def get_control(id):
             for k in update_control['alleles'].keys():
                 if update_control['alleles'][k] == 'null':
                     update_control['alleles'][k] = None
-            print ctrl
-            print update_control
             ctrl.bin_estimator_id = update_control['bin_estimator_id']
             db.session.flush()
             ctrl.initialize_alleles()
@@ -1296,7 +1275,6 @@ def get_control(id):
             handle_error(e)
     elif request.method == 'DELETE':
         ctrl = Control.query.get(id)
-        print ctrl
         db.session.delete(ctrl)
         db.session.flush()
         return jsonify(wrap_data({'status': 'Success'}))

app/microspat/fsa_extractor/PlateExtractor.py

@@ -209,7 +209,6 @@ def parallel_from_zip(cls, zip_files, ladder, color, base_size_precision, sq_lim
                           filter_parameters, scanning_parameters, creator=None, comments=None):
 
         pool = multiprocessing.Pool()
-        print "MultiProcessing Plate Loading with {} Processes".format(pool._processes)
         fn = partial(cls.from_zip_and_calculate_base_sizes, ladder=ladder, color=color,
                      base_size_precision=base_size_precision,
                      sq_limit=sq_limit,

app/microspat/models.py

@@ -6,8 +6,7 @@
 from ..custom_sql_types.custom_types import JSONEncodedData, MutableDict, MutableList, CompressedJSONEncodedData
 
 from app import db, socketio
-from app.microspat.statistics import calculate_moi
-from app.microspat.statistics.utils import calculate_prob_negative, calculate_prob_pos_if_observed
+from app.microspat.statistics import calculate_moi, calculate_prob_negative, calculate_prob_pos_if_observed
 from quantification_bias.BiasCalculator import correct_peak_proportion, calculate_beta
 from fsa_extractor.PlateExtractor import PlateExtractor, WellExtractor, ChannelExtractor
 from statistics import calculate_allele_frequencies, calculate_peak_probability
@@ -659,10 +658,7 @@ def add_sample(self, sample_id):
 
     def add_samples(self, sample_ids):
         present_sample_ids = set([_.id for _ in self.associated_samples])
-        print present_sample_ids
-        print set(sample_ids)
         full_sample_ids = list(set(sample_ids) - present_sample_ids)
-        print full_sample_ids
         n = 0
         while n * 100 < len(full_sample_ids) + 100:
             sample_ids = full_sample_ids[n * 100: (n + 1) * 100]
@@ -1327,7 +1323,7 @@ def from_peaks(cls, locus_id, peak_sets, min_artifact_peak_frequency):
             artifact_estimator = ArtifactEstimator(artifact_distance=estimator.artifact_distance,
                                                    artifact_distance_buffer=estimator.artifact_distance_buffer,
                                                    peak_data=estimator.peak_data, label=estimator.label)
-            print artifact_estimator.label
+
             for eqn in estimator.artifact_equations:
                 eventlet.sleep()
                 assert isinstance(eqn, AE.ArtifactEquation)
@@ -1679,8 +1675,6 @@ def serialize(self):
         return res
 
     def serialize_details(self):
-        print "Serializing QBE"
-        print self.locus_parameters.all()
         res = super(QuantificationBiasEstimatorProject, self).serialize_details()
         res.update({
             'locus_parameters': {_.locus_id: _.serialize() for _ in self.locus_parameters.all()},
@@ -1939,7 +1933,6 @@ def bootstrap_allele_frequencies(self):
         while alleles_changed:
 
             cycles += 1
-            print cycles
             alleles_changed = False
             allele_frequency_locus_annotations = format_locus_annotations(all_locus_annotations, peak_filters)
             allele_frequencies = calculate_allele_frequencies(allele_frequency_locus_annotations)
@@ -2055,7 +2048,6 @@ def probabilistic_peak_annotation(self, allele_frequencies=None):
             sample_annotation.moi = moi_dict[sample_annotation.id]
             locus_annotations = locus_annotation_dict[sample_annotation.id]
 
-            verbose = False
 
             for locus_annotation in locus_annotations:
                 for peak in locus_annotation.annotated_peaks:
@@ -2064,10 +2056,8 @@ def probabilistic_peak_annotation(self, allele_frequencies=None):
                     locus_annotation.annotated_peaks = calculate_prob_negative(locus_annotation.annotated_peaks,
                                                                                sample_annotation.moi,
                                                                                allele_frequencies[
-                                                                                   locus_annotation.locus.label],
-                                                                               verbose)
-                    locus_annotation.annotated_peaks = calculate_prob_pos_if_observed(locus_annotation.annotated_peaks,
-                                                                                      verbose)
+                                                                                   locus_annotation.locus.label])
+                    locus_annotation.annotated_peaks = calculate_prob_pos_if_observed(locus_annotation.annotated_peaks)
                     self.recalculate_alleles(locus_annotation)
         return self
 
@@ -2425,7 +2415,6 @@ def serialize(self):
             'min_bias_quantifier_peak_proportion': self.min_bias_quantifier_peak_proportion,
             'quantification_bias_paramters_stale': self.quantification_bias_parameters_stale
         })
-        print res
         return res
 
 
@@ -2504,7 +2493,6 @@ def copy_project_sample_annotations(cls, psa):
     def serialize(self):
         if self.sample_id and not self.sample:
             self.sample = Sample.query.get(self.sample_id)
-        print self.sample_id
         res = {
             'id': self.id,
             'sample': self.sample.serialize(),

app/microspat/quantification_bias/BiasCalculator.py

@@ -96,16 +96,11 @@ def correct_peak_proportion(beta, peak_set):
     else:
         corrected_total_peak_height = total_peak_height
 
-    print peak_set
-
-    print "Corrected Total: {}".format(corrected_total_peak_height)
-
     for peak in peak_set:
         peak_height = max(1, peak['peak_height'] - peak.get('artifact_contribution', 0))
 
         if beta:
             corrected_peak_height = peak_height * np.e ** (beta * (peak['peak_size'] - pivot_point))
-            print "Corrected Peak Height: {}".format(corrected_peak_height)
         else:
             corrected_peak_height = peak_height
 

app/microspat/statistics/__init__.py

@@ -1 +1 @@
-from utils import calculate_allele_frequencies, calculate_peak_probability, calculate_moi
+from utils import calculate_allele_frequencies, calculate_peak_probability, calculate_moi, calculate_prob_pos_if_observed, calculate_prob_negative

app/microspat/statistics/utils.py

@@ -125,21 +125,18 @@ def calculate_peak_probability(peak_set, num_possible, locus_allele_frequencies=
     return recalculated_probabilities
 
 
-def calculate_prob_pos_if_observed(peak_set, verbose=False):
+def calculate_prob_pos_if_observed(peak_set):
     for peak in peak_set:
         prob_neg = peak['prob_negative']
         prob_observed_if_neg = st.norm.sf((peak['peak_height'] - peak['artifact_contribution']) / float(max(peak['artifact_error'], 1e-6)))
 
         prob_neg_if_observed = (prob_observed_if_neg * prob_neg) / float(max(((prob_observed_if_neg * prob_neg) + 1 - prob_neg), 1e-6))
 
         peak['probability'] = max(1 - prob_neg_if_observed, 0)
-        if verbose:
-            print "Peak Height: {}   Artifact: {}   Error: {}".format(peak['peak_height'], peak['artifact_contribution'], peak['artifact_error'])
-            print "Prob neg: {}  Prob Observed if Neg: {}  Prob Neg if Observed: {}".format(prob_neg, prob_observed_if_neg, prob_neg_if_observed)
     return peak_set
 
 
-def calculate_prob_negative(peak_set, moi, allele_frequencies, verbose=False):
+def calculate_prob_negative(peak_set, moi, allele_frequencies):
 
     prob_all_perms = sum([_['probability'] * allele_frequencies.get(str(_['bin_id']), 0) for _ in peak_set]) ** moi
     if prob_all_perms < 1e-6:

app/static/plasmotrack-js/app/plasmomapper/services/sample-based-project/sample-based-project.service.ts

@@ -29,11 +29,15 @@ export class SampleBasedProjectService extends ProjectService {
         };
 
         this.getSampleLocusAnnotations = (project_id: number, sample_id: number) => {
+            console.log("Getting Sample Locus Annotations");
+            
             let url = this._locusAnnotationUrl + project_id + "/sample/" + sample_id + "/";
             return this._commonServerMethods.getList(SampleLocusAnnotation, url)
         };
 
         this.getSampleChannelAnnotations = (project_id: number, sample_id: number) => {
+            console.log("Getting Sample Channel Annotations");
+            
             let url = this._channelAnnotationsUrl + project_id + "/sample/" + sample_id;
             return this._commonServerMethods.getList(ChannelAnnotation, url)
         }

config.py

@@ -29,11 +29,24 @@ def init_app(app):
 
 class DevelopmentConfig(Config):
     DEBUG = True
-    SQLALCHEMY_DATABASE_URI = os.environ.get('DEV_DATABASE_URL') or \
+    SQLALCHEMY_TRACK_MODIFICATIONS = False
+
+    if os.name == 'nt':
+        SQLALCHEMY_DATABASE_URI = os.environ.get('DEV_DATABASE_URL') or \
+                                  'sqlite:///' + os.path.join(basedir, 'win-data-dev.sqlite')
+        CELERY_RESULT_BACKEND = os.environ.get('DEV_RESULT_BACKEND_URL') or \
+                                'db+sqlite:///' + os.path.join(basedir, 'win-result-backend-dev.sqlite')
+
+    else:
+        SQLALCHEMY_DATABASE_URI = os.environ.get('DEV_DATABASE_URL') or \
                               'sqlite:///' + os.path.join(basedir, 'data-dev.sqlite')
 
-    CELERY_RESULT_BACKEND = os.environ.get('DEV_RESULT_BACKEND_URL') or \
+        CELERY_RESULT_BACKEND = os.environ.get('DEV_RESULT_BACKEND_URL') or \
                             'db+sqlite:///' + os.path.join(basedir, 'result-backend-dev.sqlite')
+
+    # SQLALCHEMY_DATABASE_URI = 'sqlite:///' + os.path.join(basedir, 'data-dev.sqlite')
+    # CELERY_RESULT_BACKEND = 'db+sqlite:///' + os.path.join(basedir, 'result-backend-dev.sqlite')
+
     SQLALCHEMY_ECHO = False
     LOGGING_LEVEL = logging.DEBUG