Merge pull request #379 from moevm/check_pipeline_fixes

Check pipeline / tables / export fixes

Author: HadronCollider

app/db/db_methods.py

@@ -5,8 +5,8 @@
 from bson import ObjectId
 from gridfs import GridFSBucket, NoFile
 from pymongo import MongoClient
-
 from utils import convert_to
+
 from .db_types import User, Presentation, Check, Consumers, Logs
 
 client = MongoClient("mongodb://mongodb:27017")
@@ -133,11 +133,6 @@ def delete_presentation(user, presentation_id):
         return user, get_presentation(presentation_id)
 
 
-# Creates checks from given user check-list (not created in DB)
-def create_check(user, file_type='pres'):
-    return Check({'enabled_checks': user.criteria, 'file_type': file_type})
-
-
 # Adds checks to given presentations, updates presentations, returns presentations and checks id
 def add_check(file_id, check):
     checks_id = checks_collection.insert_one(check.pack()).inserted_id
@@ -215,7 +210,7 @@ def set_passbacked_flag(checks_id, flag):
     upd_check['$set']['is_passbacked'] = flag
     if not (flag is None):
         # None - if user without passback
-        upd_check['$set']['lms_passback_time'] = datetime.now()
+        upd_check['$set']['lms_passback_time'] = None
     check = checks_collection.update_one({'_id': checks_id}, upd_check)
     return check if check else None
 

app/db/db_types.py

@@ -96,6 +96,8 @@ def __init__(self, dictionary=None):
         self.score = dictionary.get('score', -1)
         self.file_type = dictionary.get('file_type', DEFAULT_TYPE_INFO)
         self.enabled_checks = dictionary.get('enabled_checks', BASE_PACKS.get(self.file_type['type']).name)
+        self.criteria = dictionary.get('criteria', BASE_PACKS.get(self.file_type['type']).name)
+        self.params_for_passback = dictionary.get('params_for_passback', None)
         self.is_failed = dictionary.get('is_failed', None)
         self.is_ended = dictionary.get('is_ended', True)
         self.is_passed = dictionary.get('is_passed', int(self.score) == 1)

app/main/checker.py

@@ -1,24 +1,23 @@
 from db.db_methods import get_criteria_pack
+
 from .check_packs import BaseCriterionPack
 
 
-def check(parsed_file, check_obj, filename, user):
+def check(parsed_file, check_obj):
+    filename = check_obj.filename
     file_info = {
         'file': parsed_file,
         'filename': filename,
         'pdf_id': check_obj.conv_pdf_fs_id
     }
-    pack = BaseCriterionPack(**get_criteria_pack(user.criteria))
+    pack = BaseCriterionPack(**get_criteria_pack(check_obj.criteria))
     pack.init(file_info)
     result, score, is_passed = pack.check()
 
     check_obj.enabled_checks = result
     check_obj.score = score
     check_obj.is_passed = is_passed
-    check_obj.filename = filename
-    check_obj.user = user.username
-    check_obj.lms_user_id = user.lms_user_id
-    if user.params_for_passback:
+    if check_obj.params_for_passback:
         check_obj.is_passbacked = False
 
     return check_obj

app/passback_grades.py

@@ -1,38 +1,56 @@
 import configparser
 
+import urllib3
 
-from db.db_methods import ConsumersDBManager, get_unpassed_checks, set_passbacked_flag, get_user
+from db.db_methods import ConsumersDBManager, get_unpassed_checks, set_passbacked_flag
+from lti_session_passback.lti_provider import LTIProvider
 from root_logger import get_root_logger
 from utils import RepeatedTimer
-from lti_session_passback.lti_provider import LTIProvider
 
+urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
 config = configparser.ConfigParser()
 config.read('app/config.ini')
 
 logger = get_root_logger('passback_grades')
 
 
+def check_success_response(response):
+    return response.code_major == 'success' and response.severity == 'status'
+
+
 class ChecksPassBack:
     def __init__(self, timeout_seconds=10):
         self._timeout_seconds = timeout_seconds
 
     def grade_passback(self, check):
-        user = check.get('user')
-        passback_params = get_user(user).params_for_passback
+        passback_params = check.get('params_for_passback', None)
         if not passback_params or passback_params["lis_outcome_service_url"] == "lis_outcome_service_url":
             set_passbacked_flag(check.get('_id'), None)
             return
 
         consumer_secret = ConsumersDBManager.get_secret(passback_params['oauth_consumer_key'])
-        response = LTIProvider.from_unpacked_request(secret=consumer_secret, params=passback_params, headers=None,
-                                                      url=None).post_replace_result(score=check.get('score'))
+        provider = LTIProvider.from_unpacked_request(secret=consumer_secret, params=passback_params, headers=None,
+                                                     url=None)
 
-        if response.code_major == 'success' and response.severity == 'status':
-            logger.info('Score was successfully passed back: score = {}, check_id = {}'.format(check.get('score'),
-                                                                                               check.get('_id')))
-            set_passbacked_flag(check.get('_id'), True)
+        current_lms_result = provider.post_read_result()
+        current_lms_score = float(current_lms_result.score) if check_success_response(current_lms_result) else 0.0
+        system_score = check.get('score')
+
+        if round(system_score, 2) > current_lms_score:
+            # if our score > lms_score -> send, else - ???
+            response = provider.post_replace_result(score=check.get('score'))
+
+            if check_success_response(response):
+                logger.info('Score was successfully passed back: score = {}, check_id = {}'.format(check.get('score'),
+                                                                                                   check.get('_id')))
+                set_passbacked_flag(check.get('_id'), True)
+            else:
+                logger.error('Passback failed for check_id = {}'.format(check.get('_id')))
         else:
-            logger.error('Passback failed for check_id = {}'.format(check.get('_id')))
+            logger.info(
+                'LMS score is more then current (not passbacked): LMS_score = {}, system_score = {} check_id = {}'.format(
+                    current_lms_score, check.get('score'), check.get('_id')))
+            set_passbacked_flag(check.get('_id'), None)
 
     def _run(self):
         logger.info('Start passback')

app/server.py

@@ -17,6 +17,7 @@
 from flask_recaptcha import ReCaptcha
 
 import servants.user as user
+from app.utils import format_check_for_table
 from db import db_methods
 from db.db_types import Check
 from lti_session_passback.lti import utils
@@ -26,7 +27,7 @@
 from root_logger import get_logging_stdout_handler, get_root_logger
 from servants import pre_luncher
 from tasks import create_task
-from utils import checklist_filter, decorator_assertion, get_file_len, timezone_offset, format_check
+from utils import checklist_filter, decorator_assertion, get_file_len, format_check
 
 logger = get_root_logger('web')
 UPLOAD_FOLDER = '/usr/src/project/files'
@@ -35,7 +36,7 @@
     'report': {'doc', 'odt', 'docx'}
 }
 DOCUMENT_TYPES = {'Лабораторная работа', 'Курсовая работа', 'ВКР'}
-TABLE_COLUMNS = ['Solution', 'User', 'File', 'Check added', 'LMS date', 'Score']
+TABLE_COLUMNS = ['Solution', 'User', 'File', 'Criteria', 'Check added', 'LMS date', 'Score']
 
 app = Flask(__name__, static_folder="./../src/", template_folder="./templates/")
 app.config.from_pyfile('settings.py')
@@ -188,11 +189,13 @@ def run_task():
         'user': current_user.username,
         'lms_user_id': current_user.lms_user_id,
         'enabled_checks': current_user.criteria,
+        'criteria': current_user.criteria,
         'file_type': current_user.file_type,
         'filename': file.filename,
         'score': -1,  # score=-1 -> checking in progress
         'is_ended': False,
-        'is_failed': False
+        'is_failed': False,
+        'params_for_passback': current_user.params_for_passback
     })
     db_methods.add_check(file_id, check)  # add check for parsed_file to db
     task = create_task.delay(check.pack(to_str=True))  # add check to queue
@@ -243,7 +246,7 @@ def results(_id):
     if check is not None:
         # show processing time for user
         avg_process_time = None if check.is_ended else db_methods.get_average_processing_time()
-        return render_template("./results.html", navi_upload=True, name=current_user.name, results=check, id=_id,
+        return render_template("./results.html", navi_upload=True, results=check,
                                columns=TABLE_COLUMNS, avg_process_time=avg_process_time,
                                stats=format_check(check.pack()))
     else:
@@ -336,7 +339,8 @@ def api_criteria_pack():
     #  testing pack initialization
     file_type_info = {'type': file_type}
     if file_type == DEFAULT_REPORT_TYPE_INFO['type']:
-        file_type_info['report_type'] = report_type if report_type in REPORT_TYPES else DEFAULT_REPORT_TYPE_INFO['report_type']
+        file_type_info['report_type'] = report_type if report_type in REPORT_TYPES else DEFAULT_REPORT_TYPE_INFO[
+            'report_type']
     inited, err = init_criterions(raw_criterions, file_type=file_type_info)
     if len(raw_criterions) != len(inited) or err:
         msg = f"При инициализации набора {pack_name} возникли ошибки. JSON-конфигурация: '{raw_criterions}'. Успешно инициализированные: {inited}. Возникшие ошибки: {err}."
@@ -416,16 +420,7 @@ def check_list_data():
     # construct response
     response = {
         "total": count,
-        "rows": [{
-            "_id": str(item["_id"]),
-            "filename": item["filename"],
-            "user": item["user"],
-            "lms-user-id": item["lms_user_id"] if item.get("lms_user_id") else '-',
-            "upload-date": (item["_id"].generation_time + timezone_offset).strftime("%d.%m.%Y %H:%M:%S"),
-            "moodle-date": item['lms_passback_time'].strftime("%d.%m.%Y %H:%M:%S") if item.get(
-                'lms_passback_time') else '-',
-            "score": item["score"]
-        } for item in rows]
+        "rows": [format_check_for_table(item) for item in rows]
     }
 
     # return json data
@@ -448,17 +443,7 @@ def get_query(req):
 
 def get_stats():
     rows, count = db_methods.get_checks(**get_query(request))
-    return [{
-        "_id": str(item["_id"]),
-        "filename": item["filename"],
-        "user": item["user"],
-        "lms-username": item["user"].rsplit('_', 1)[0],
-        "lms-user-id": item["lms_user_id"] if item.get("lms_user_id") else '-',
-        "upload-date": (item["_id"].generation_time + timezone_offset).strftime("%d.%m.%Y %H:%M:%S"),
-        "moodle-date": item['lms_passback_time'].strftime("%d.%m.%Y %H:%M:%S") if item.get(
-            'lms_passback_time') else '-',
-        "score": item["score"]
-    } for item in rows]
+    return [format_check_for_table(item) for item in rows]
 
 
 @app.route("/get_csv")

app/tasks.py

@@ -29,7 +29,7 @@ def create_task(self, check_info):
     pdf_filepath = join(FILES_FOLDER, f"{check_id}.pdf")
     try:
         user = get_user(check_obj.user)
-        updated_check = check(parse(original_filepath), check_obj, check_obj.filename, user)
+        updated_check = check(parse(original_filepath), check_obj)
         updated_check.is_ended = True
         db_methods.update_check(updated_check)  # save to db
         db_methods.mark_celery_task_as_finished(self.request.id)

app/templates/check_list.html

@@ -8,12 +8,15 @@
 
 {% block main %}
     <style>
-        .fht-cell { margin: 0 0.2rem 0.2rem; }
+        .fht-cell {
+            margin: 0 0.2rem 0.2rem;
+        }
+
         /* .check-list-table th td { border: #54585d; } */
 
         #check-list-table {
             table-layout: fixed;
-            word-wrap:break-word;
+            word-wrap: break-word;
         }
 
         #check-list-table.table-bordered,
@@ -26,41 +29,48 @@
     <div class="holder row">
         <div class="container-fluid table-responsive-xl">
             <table id="check-list-table" class="table bg-white"
-                data-filter-control="true"
-                data-pagination="true"
-                data-page-list="[5, 10, 25, 50, All]"
-                data-ajax="ajaxRequest"
-                data-query-params="queryParams"
-                data-show-refresh="true"
-                data-auto-refresh="true"
-                data-auto-refresh-interval="10"
-                data-side-pagination="server"
-                data-icon-size="lg"
-                data-buttons="buttons"
-                data-show-filter-control-switch="true"
-                data-show-button-icons="false"
-                data-show-button-text="true"
+                   data-filter-control="true"
+                   data-pagination="true"
+                   data-page-list="[5, 10, 25, 50, All]"
+                   data-ajax="ajaxRequest"
+                   data-query-params="queryParams"
+                   data-show-refresh="true"
+                   data-auto-refresh="true"
+                   data-auto-refresh-interval="10"
+                   data-side-pagination="server"
+                   data-icon-size="lg"
+                   data-buttons="buttons"
+                   data-show-filter-control-switch="true"
+                   data-show-button-icons="false"
+                   data-show-button-text="true"
             >
                 <thead>
-                    <tr>
-                        <th data-field="_id" data-width="160">Upload ID</th>
-                        <th data-field="filename" data-filter-control="input" data-sortable="true">File name</th>
-                        {% if current_user.is_admin %}
-                            <th data-field="user" data-filter-control="input" data-sortable="true">Username</th>
-                            <th data-field="lms-user-id" data-sortable="false">User ID</th>
-                        {% endif %}
-                        <th data-field="upload-date" data-filter-control="datepicker" data-filter-datepicker-options='{"autoclose":true, "clearBtn": true, "multidate":2, "multidateSeparator":"-", "format":"dd.mm.yyyy"}' data-sortable="true">Upload date</th>
-                        <th data-field="moodle-date" data-filter-control="datepicker" data-filter-datepicker-options='{"autoclose":true, "clearBtn": true, "multidate":2, "multidateSeparator":"-", "format":"dd.mm.yyyy"}' data-sortable="true">Moodle upload date</th>
-                        <th data-field="score" data-filter-control="input" data-sortable="true">Result</th>
-                    </tr>
+                <tr>
+                    <th data-field="_id" data-width="160">Upload ID</th>
+                    <th data-field="filename" data-filter-control="input" data-sortable="true">File name</th>
+                    <th data-field="criteria" data-sortable="true">Criteria pack</th>
+                    {% if current_user.is_admin %}
+                        <th data-field="user" data-filter-control="input" data-sortable="true">Username</th>
+                        <th data-field="lms-user-id" data-sortable="false">User ID</th>
+                    {% endif %}
+                    <th data-field="upload-date" data-filter-control="datepicker"
+                        data-filter-datepicker-options='{"autoclose":true, "clearBtn": true, "multidate":2, "multidateSeparator":"-", "format":"dd.mm.yyyy"}'
+                        data-sortable="true">Upload date
+                    </th>
+                    <th data-field="moodle-date" data-filter-control="datepicker"
+                        data-filter-datepicker-options='{"autoclose":true, "clearBtn": true, "multidate":2, "multidateSeparator":"-", "format":"dd.mm.yyyy"}'
+                        data-sortable="true">Moodle upload date
+                    </th>
+                    <th data-field="score" data-filter-control="input" data-sortable="true">Result</th>
+                </tr>
                 </thead>
             </table>
         </div>
     </div>
 {% endblock main %}
 
 {% block script %}
-<script>
-    var is_admin = "{{current_user.is_admin}}" == "True"   
-</script>
+    <script>
+        var is_admin = "{{current_user.is_admin}}" == "True"
+    </script>
 {% endblock %}