Merge pull request #239 from FIRST-Tech-Challenge/pr_issue_237

Fixed issue with creating a dataset with more than 42 videos

Author: lizlooney

server/app_engine.py

@@ -837,10 +837,10 @@ def can_delete_videos():
     team_uuid = team_info.retrieve_team_uuid(flask.session, flask.request)
     data = validate_keys(flask.request.form.to_dict(flat=True),
         ['video_uuids'])
-    video_uuids_json = storage.validate_uuids_json(data.get('video_uuids'))
+    video_uuids = storage.validate_uuids_json(data.get('video_uuids'))
     # storage.can_delete_videos will raise HttpErrorNotFound
     # if any of the team_uuid/video_uuid is not found.
-    can_delete_videos, messages = storage.can_delete_videos(team_uuid, video_uuids_json)
+    can_delete_videos, messages = storage.can_delete_videos(team_uuid, video_uuids)
     response = {
         'can_delete_videos': can_delete_videos,
         'messages': messages,
@@ -1068,7 +1068,7 @@ def prepare_to_start_dataset_production():
             'message': 'The Description is not valid or is a duplicate.'
         }
         return flask.jsonify(__sanitize(response))
-    video_uuids_json = storage.validate_uuids_json(data.get('video_uuids'))
+    video_uuids = storage.validate_uuids_json(data.get('video_uuids'))
     try:
         # The following min/max number (0 and 90) should match the min/max values in root.html.
         eval_percent = validate_float(data.get('eval_percent'), min=0, max=90)
@@ -1093,9 +1093,9 @@ def prepare_to_start_dataset_production():
     # dataset_producer.prepare_to_start_dataset_production will raise HttpErrorNotFound
     # if any of the team_uuid/video_uuids is not found or if none of the videos have labeled frames.
     dataset_uuid = dataset_producer.prepare_to_start_dataset_production(
-        team_uuid, description, video_uuids_json, eval_percent, create_time_ms)
+        team_uuid, description, video_uuids, eval_percent, create_time_ms)
     action_parameters = dataset_producer.make_action_parameters(
-        team_uuid, dataset_uuid, video_uuids_json, eval_percent, create_time_ms)
+        team_uuid, dataset_uuid, video_uuids, eval_percent, create_time_ms)
     action.trigger_action_via_blob(action_parameters)
     response = {
         'dataset_uuid': dataset_uuid,
@@ -1147,10 +1147,10 @@ def can_delete_datasets():
     team_uuid = team_info.retrieve_team_uuid(flask.session, flask.request)
     data = validate_keys(flask.request.form.to_dict(flat=True),
         ['dataset_uuids'])
-    dataset_uuids_json = storage.validate_uuids_json(data.get('dataset_uuids'))
+    dataset_uuids = storage.validate_uuids_json(data.get('dataset_uuids'))
     # storage.can_delete_datasets will raise HttpErrorNotFound
     # if any of the team_uuid/dataset_uuid is not found.
-    can_delete_datasets, messages = storage.can_delete_datasets(team_uuid, dataset_uuids_json)
+    can_delete_datasets, messages = storage.can_delete_datasets(team_uuid, dataset_uuids)
     response = {
         'can_delete_datasets': can_delete_datasets,
         'messages': messages,
@@ -1245,7 +1245,7 @@ def start_training_model():
     # description = validate_description(data.get('description'),
     #        other_descriptions=[m['description'] for m in storage.retrieve_model_list(team_uuid)])
     description = validate_description(data.get('description'))
-    dataset_uuids_json = storage.validate_uuids_json(data.get('dataset_uuids'))
+    dataset_uuids = storage.validate_uuids_json(data.get('dataset_uuids'))
     starting_model = model_trainer.validate_starting_model(data.get('starting_model'))
     max_running_minutes = validate_positive_float(data.get('max_running_minutes'))
     num_training_steps = validate_int(data.get('num_training_steps'),
@@ -1259,7 +1259,7 @@ def start_training_model():
     # if the sorted_label_list values for all the datasets are not the same.
     # model_trainer.start_training_model will raise HttpErrorUnprocessableEntity
     # if the max_running_minutes exceeds the team's remaining_training_minutes.
-    model_entity = model_trainer.start_training_model(team_uuid, description, dataset_uuids_json,
+    model_entity = model_trainer.start_training_model(team_uuid, description, dataset_uuids,
         starting_model, max_running_minutes, num_training_steps, create_time_ms, config.config[KEY_USE_TPU])
     # Retrieve the team entity so the client gets the updated remaining_training_minutes.
     team_entity = storage.retrieve_team_entity(team_uuid)
@@ -1427,10 +1427,10 @@ def can_delete_models():
     team_uuid = team_info.retrieve_team_uuid(flask.session, flask.request)
     data = validate_keys(flask.request.form.to_dict(flat=True),
         ['model_uuids'])
-    model_uuids_json = storage.validate_uuids_json(data.get('model_uuids'))
+    model_uuids = storage.validate_uuids_json(data.get('model_uuids'))
     # storage.can_delete_models will raise HttpErrorNotFound
     # if any of the team_uuid/model_uuid is not found.
-    can_delete_models, messages = storage.can_delete_models(team_uuid, model_uuids_json)
+    can_delete_models, messages = storage.can_delete_models(team_uuid, model_uuids)
     response = {
         'can_delete_models': can_delete_models,
         'messages': messages,

server/blob_storage.py

@@ -16,7 +16,6 @@
 
 # Python Standard Library
 from datetime import datetime, timedelta
-import json
 import logging
 import re
 import time

server/dataset_producer.py

@@ -50,39 +50,42 @@
     'video_filename', 'frame_number', 'filename', 'image', 'format', 'bboxes_text'])
 
 
-def prepare_to_start_dataset_production(team_uuid, description, video_uuids_json, eval_percent, create_time_ms):
+def prepare_to_start_dataset_production(team_uuid, description, video_uuid_list, eval_percent, create_time_ms):
     # storage.prepare_to_start_dataset_production will raise HttpErrorNotFound
     # if any of the team_uuid/video_uuids is not found or if none of the videos have labeled frames.
     dataset_uuid = storage.prepare_to_start_dataset_production(team_uuid, description,
-        json.loads(video_uuids_json), eval_percent, create_time_ms)
+        video_uuid_list, eval_percent, create_time_ms)
     return dataset_uuid
 
-def make_action_parameters(team_uuid, dataset_uuid, video_uuids_json, eval_percent, create_time_ms):
+def make_action_parameters(team_uuid, dataset_uuid, video_uuid_list, eval_percent, create_time_ms):
     action_parameters = action.create_action_parameters(
         team_uuid, action.ACTION_NAME_DATASET_PRODUCE)
     action_parameters['team_uuid'] = team_uuid
     action_parameters['dataset_uuid'] = dataset_uuid
-    action_parameters['video_uuids_json'] = video_uuids_json
+    action_parameters['video_uuid_list'] = video_uuid_list
     action_parameters['eval_percent'] = eval_percent
     action_parameters['create_time_ms'] = create_time_ms
     return action_parameters
 
 def produce_dataset(action_parameters):
     team_uuid = action_parameters['team_uuid']
     dataset_uuid = action_parameters['dataset_uuid']
-    video_uuids_json = action_parameters['video_uuids_json']
+    if 'video_uuids_json' in action_parameters:
+        video_uuids_json = action_parameters['video_uuids_json']
+        video_uuid_list = json.loads(video_uuids_json)
+    else:
+        video_uuid_list = action_parameters['video_uuid_list']
     eval_percent = action_parameters['eval_percent']
     create_time_ms = action_parameters['create_time_ms']
 
-    video_uuid_list = json.loads(video_uuids_json)
     if len(video_uuid_list) == 0:
         message = "Error: No videos to process."
         logging.critical(message)
         raise exceptions.HttpErrorBadRequest(message)
 
     video_entities = storage.retrieve_video_entities(team_uuid, video_uuid_list)
     if len(video_entities) != len(video_uuid_list):
-        message = 'Error: One or more videos not found for video_uuids=%s.' % video_uuids_json
+        message = 'Error: One or more videos not found for video_uuids=%s.' % str(video_uuid_list)
         logging.critical(message)
         raise exceptions.HttpErrorNotFound(message)
 

server/model_trainer.py

@@ -129,7 +129,7 @@ def __get_num_warmup_steps(original_starting_model, checkpoint_every_n, num_trai
     starting_model_data = STARTING_MODELS[original_starting_model]
     return min(starting_model_data['num_warmup_steps'], num_training_steps)
 
-def start_training_model(team_uuid, description, dataset_uuids_json,
+def start_training_model(team_uuid, description, dataset_uuid_list,
         starting_model, max_running_minutes, num_training_steps, create_time_ms, use_tpu):
     found_starting_model = starting_model in STARTING_MODELS
     if found_starting_model:
@@ -167,10 +167,9 @@ def start_training_model(team_uuid, description, dataset_uuids_json,
         if fine_tune_checkpoint.endswith('.index'):
             fine_tune_checkpoint = fine_tune_checkpoint[:-6]
 
-    dataset_uuid_list = json.loads(dataset_uuids_json)
     dataset_entities = storage.retrieve_dataset_entities(team_uuid, dataset_uuid_list)
     if len(dataset_entities) != len(dataset_uuid_list):
-        message = 'Error: One or more datasets not found for dataset_uuids=%s.' % dataset_uuids_json
+        message = 'Error: One or more datasets not found for dataset_uuids=%s.' % str(dataset_uuid_list)
         logging.critical(message)
         raise exceptions.HttpErrorNotFound(message)
 

server/storage.py

@@ -63,13 +63,14 @@ def validate_uuid(s):
 
 def validate_uuids_json(s):
     try:
-        for u in json.loads(s):
+        l = json.loads(s)
+        for u in l:
             validate_uuid(u)
     except:
         message = "Error: '%s' is not a valid argument." % s
         logging.critical(message)
         raise exceptions.HttpErrorBadRequest(message)
-    return s
+    return l
 
 # teams - public methods
 
@@ -392,10 +393,9 @@ def retrieve_video_entity_for_labeling(team_uuid, video_uuid):
                     transaction.delete(tracker_client_entity.key)
         return video_entity
 
-def can_delete_videos(team_uuid, video_uuids_json):
+def can_delete_videos(team_uuid, video_uuid_requested_list):
     can_delete_videos = True
     messages = []
-    video_uuid_requested_list = json.loads(video_uuids_json)
     all_video_entities = retrieve_video_list(team_uuid)
     # Build a list of the video uuids that we found.
     video_uuids_found_list = []
@@ -1657,10 +1657,9 @@ def retrieve_model_list(team_uuid):
             model_entity['model_folder'] = blob_storage.get_old_model_folder(team_uuid, model_entity['model_uuid'])
     return model_entities
 
-def can_delete_datasets(team_uuid, dataset_uuids_json):
+def can_delete_datasets(team_uuid, dataset_uuid_requested_list):
     can_delete_datasets = True
     messages = []
-    dataset_uuid_requested_list = json.loads(dataset_uuids_json)
     all_dataset_entities = retrieve_dataset_list(team_uuid)
     # Build a list of the dataset uuids that we found.
     dataset_uuids_found_list = []
@@ -1704,10 +1703,9 @@ def can_delete_datasets(team_uuid, dataset_uuids_json):
                 messages.append(message)
     return can_delete_datasets, messages
 
-def can_delete_models(team_uuid, model_uuids_json):
+def can_delete_models(team_uuid, model_uuid_requested_list):
     can_delete_models = True
     messages = []
-    model_uuid_requested_list = json.loads(model_uuids_json)
     all_model_entities = retrieve_model_list(team_uuid)
     # Build a list of the model uuids that we found.
     model_uuids_found_list = []