chg: [tracker] add tracked objects status: viewed, done, rejected (fp) + improve invalid yara-rule error message

Author: Terrtia

bin/lib/Tracker.py

@@ -416,6 +416,12 @@ def get_meta(self, options):
             meta['description'] = self.get_description()
         if 'nb_objs' in options:
             meta['nb_objs'] = self.get_nb_objs()
+        if 'objs_stats' in options:
+            if 'nb_objs' in meta:
+                total = meta['nb_objs']
+            else:
+                total = None
+            meta['objs_stats'] = self.get_objs_stats(total=total)
         if 'tags' in options:
             meta['tags'] = self.get_tags()
         if 'filters' in options:
@@ -456,6 +462,12 @@ def get_nb_objs(self):
                 objs[obj_type] = nb
         return objs
 
+    def get_nb_total_objs(self):
+        nb = 0
+        for obj_type in get_objects_tracked():
+            nb += self.get_nb_objs_by_type(obj_type)
+        return nb
+
     def get_objs(self):
         objs = []
         for obj_type in get_objects_tracked():
@@ -488,6 +500,9 @@ def get_objs_by_daterange(self, date_from, date_to, obj_types):
     def get_obj_dates(self, obj_type, subtype, obj_id):
         return r_tracker.smembers(f'obj:tracker:{obj_type}:{subtype}:{obj_id}:{self.uuid}')
 
+    def is_tracked_obj(self, obj_gid):
+        return r_tracker.sismember(f'obj:trackers:{obj_gid}', self.uuid)
+
     # - TODO Data Retention TO Implement - #
     # Or Daily/Monthly Global DB Cleanup:
     #    Iterate on each tracker:
@@ -516,15 +531,95 @@ def add(self, obj_type, subtype, obj_id, date=None):
     def remove(self, obj_type, subtype, obj_id):
         if not subtype:
             subtype = ''
+        obj_gid = f'{obj_type}:{subtype}:{obj_id}'
 
         for date in self.get_obj_dates(obj_type, subtype, obj_id):
-            r_tracker.srem(f'tracker:objs:{self.uuid}:{date}', f'{obj_type}:{subtype}:{obj_id}')
+            r_tracker.srem(f'tracker:objs:{self.uuid}:{date}', obj_gid)
             r_tracker.srem(f'obj:tracker:{obj_type}:{subtype}:{obj_id}:{self.uuid}', date)
 
         r_tracker.srem(f'obj:trackers:{obj_type}:{subtype}:{obj_id}', self.uuid)
         r_tracker.srem(f'tracker:objs:{self.uuid}:{obj_type}', f'{subtype}:{obj_id}')
+        # obj status
+        self.delete_obj_status(obj_gid)
+
         self.update_daterange()
 
+    def get_nb_objs_read(self):
+        return r_tracker.scard(f'tracker:objs:read:{self.uuid}')
+
+    def get_objs_done(self):
+        return r_tracker.smembers(f'tracker:objs:done:{self.uuid}')
+
+    def get_nb_objs_done(self):
+        return r_tracker.scard(f'tracker:objs:done:{self.uuid}')
+
+    def get_objs_rejected(self):
+        return r_tracker.smembers(f'tracker:objs:fp:{self.uuid}')
+
+    def get_nb_objs_rejected(self):
+        return r_tracker.scard(f'tracker:objs:fp:{self.uuid}')
+
+    def get_objs_stats(self, total=None):
+        done = self.get_nb_objs_done()
+        fp = self.get_nb_objs_rejected()
+        read = self.get_nb_objs_read()
+        if total:
+            nb = 0
+            for nb_obj in total:
+                nb += total[nb_obj]
+        else:
+            nb = self.get_nb_total_objs()
+        unread = nb - done - fp - read
+        return {'done': done, 'fp': fp, 'read': read, 'unread': unread}
+
+    def is_obj_read(self, obj_gid):
+        return r_tracker.sismember(f'tracker:objs:read:{self.uuid}', obj_gid)
+
+    def is_obj_done(self, obj_gid):
+        return r_tracker.sismember(f'tracker:objs:done:{self.uuid}', obj_gid)
+
+    def is_obj_rejected(self, obj_gid):
+        return r_tracker.sismember(f'tracker:objs:fp:{self.uuid}', obj_gid)
+
+    def get_obj_status(self, obj_gid):
+        if self.is_obj_read(obj_gid):
+            return 'read'
+        elif self.is_obj_done(obj_gid):
+            return 'done'
+        elif self.is_obj_rejected(obj_gid):
+            return 'rejected'
+        else:
+            return 'unread'
+
+    def obj_read(self, obj_gid):
+        self.obj_undone(obj_gid)
+        self.obj_unreject(obj_gid)
+        r_tracker.sadd(f'tracker:objs:read:{self.uuid}', obj_gid)
+
+    def obj_unread(self, obj_gid):
+        r_tracker.srem(f'tracker:objs:read:{self.uuid}', obj_gid)
+
+    def obj_done(self, obj_gid):
+        self.obj_unread(obj_gid)
+        self.obj_unreject(obj_gid)
+        r_tracker.sadd(f'tracker:objs:done:{self.uuid}', obj_gid)
+
+    def obj_undone(self, obj_gid):
+        r_tracker.srem(f'tracker:objs:done:{self.uuid}', obj_gid)
+
+    def obj_reject(self, obj_gid):
+        self.obj_unread(obj_gid)
+        self.obj_undone(obj_gid)
+        r_tracker.sadd(f'tracker:objs:fp:{self.uuid}', obj_gid)
+
+    def obj_unreject(self, obj_gid):
+        r_tracker.srem(f'tracker:objs:fp:{self.uuid}', obj_gid)
+
+    def delete_obj_status(self, obj_gid):
+        self.obj_unread(obj_gid)
+        self.obj_undone(obj_gid)
+        self.obj_unreject(obj_gid)
+
     # TODO escape custom tags
     # TODO escape mails ????
     def create(self, tracker_type, to_track, org, user_id, level, description=None, filters={}, tags=[], mails=[], webhook=None):
@@ -730,6 +825,10 @@ def delete(self):
         ail_orgs.remove_obj_to_org(self.get_org(), 'tracker', self.uuid)
         # meta
         r_tracker.delete(f'tracker:{self.uuid}')
+        # objs status
+        r_tracker.delete(f'tracker:objs:read:{self.uuid}')
+        r_tracker.delete(f'tracker:objs:done:{self.uuid}')
+        r_tracker.delete(f'tracker:objs:fp:{self.uuid}')
         trigger_trackers_refresh(tracker_type)
 
 
@@ -932,6 +1031,9 @@ def is_obj_tracked(obj_type, subtype, obj_id):
 def get_obj_trackers(obj_type, subtype, obj_id):
     return r_tracker.smembers(f'obj:trackers:{obj_type}:{subtype}:{obj_id}')
 
+def set_obj_tracker_read(tracker_uuid, obj_gid):
+    r_tracker.sadd(f'tracker:objs:read:{tracker_uuid}', obj_gid)
+
 def delete_obj_trackers(obj_type, subtype, obj_id):
     for tracker_uuid in get_obj_trackers(obj_type, subtype, obj_id):
         tracker = Tracker(tracker_uuid)
@@ -1081,8 +1183,9 @@ def api_validate_tracker_to_add(to_track, tracker_type, nb_words=1):
             return {"status": "error", "reason": "Invalid domain name"}, 400
 
     elif tracker_type == 'yara_custom':
-        if not is_valid_yara_rule(to_track):
-            return {"status": "error", "reason": "Invalid custom Yara Rule"}, 400
+        valid_yara_rule, error = is_valid_yara_rule(to_track)
+        if not valid_yara_rule:
+            return {"status": "error", "reason": f"Invalid Yara Rule: {error}"}, 400
     elif tracker_type == 'yara_default':
         if not is_valid_default_yara_rule(to_track):
             return {"status": "error", "reason": "The Yara Rule doesn't exist"}, 400
@@ -1282,6 +1385,54 @@ def api_tracker_remove_object(data, user_org, user_id, user_role):
         return {"status": "error", "reason": "Invalid Object"}, 400
     return tracker.remove(obj_type, subtype, obj_id), 200
 
+def api_tracker_object_status_done(data, user_org, user_id, user_role):
+    tracker_uuid = data.get('uuid')
+    res = api_check_tracker_acl(tracker_uuid, user_org, user_id, user_role, 'edit')
+    if res:
+        return res
+
+    tracker = Tracker(tracker_uuid)
+    object_gid = data.get('gid')
+    if not tracker.is_tracked_obj(object_gid):
+        return {"status": "error", "reason": "Not Tracked Object"}, 404
+    return tracker.obj_done(object_gid), 200
+
+def api_tracker_object_status_reject(data, user_org, user_id, user_role):
+    tracker_uuid = data.get('uuid')
+    res = api_check_tracker_acl(tracker_uuid, user_org, user_id, user_role, 'edit')
+    if res:
+        return res
+
+    tracker = Tracker(tracker_uuid)
+    object_gid = data.get('gid')
+    if not tracker.is_tracked_obj(object_gid):
+        return {"status": "error", "reason": "Not Tracked Object"}, 404
+    return tracker.obj_reject(object_gid), 200
+
+def api_tracker_object_status_unread(data, user_org, user_id, user_role):
+    tracker_uuid = data.get('uuid')
+    res = api_check_tracker_acl(tracker_uuid, user_org, user_id, user_role, 'edit')
+    if res:
+        return res
+
+    tracker = Tracker(tracker_uuid)
+    object_gid = data.get('gid')
+    if not tracker.is_tracked_obj(object_gid):
+        return {"status": "error", "reason": "Not Tracked Object"}, 404
+    return tracker.delete_obj_status(object_gid), 200
+
+def api_tracker_object_status_read(data, user_org, user_id, user_role):
+    tracker_uuid = data.get('uuid')
+    res = api_check_tracker_acl(tracker_uuid, user_org, user_id, user_role, 'edit')
+    if res:
+        return res
+
+    tracker = Tracker(tracker_uuid)
+    object_gid = data.get('gid')
+    if not tracker.is_tracked_obj(object_gid):
+        return {"status": "error", "reason": "Not Tracked Object"}, 404
+    return tracker.obj_read(object_gid), 200
+
 ## -- CREATE TRACKER -- ##
 
 ####################
@@ -1411,9 +1562,9 @@ def reload_yara_rules():
 def is_valid_yara_rule(yara_rule):
     try:
         yara.compile(source=yara_rule)
-        return True
-    except:
-        return False
+        return True, None
+    except Exception as e:
+        return False, str(e)
 
 def is_default_yara_rule(tracked_yara_name):
     yara_dir = get_yara_rules_dir()
@@ -2000,8 +2151,9 @@ def api_resume_retro_hunt_task(user_org, user_id, user_role, task_uuid):
 
 def api_validate_rule_to_add(rule, rule_type):
     if rule_type == 'yara_custom':
-        if not is_valid_yara_rule(rule):
-            return {"status": "error", "reason": "Invalid custom Yara Rule"}, 400
+        valid_yara_rule, error = is_valid_yara_rule(rule)
+        if not valid_yara_rule:
+            return {"status": "error", "reason": f"Invalid Yara Rule: {e}"}, 400
     elif rule_type == 'yara_default':
         if not is_valid_default_yara_rule(rule):
             return {"status": "error", "reason": "The Yara Rule doesn't exist"}, 400

var/www/blueprints/hunters.py

@@ -200,7 +200,7 @@ def show_tracker():
     tracker = Tracker.Tracker(tracker_uuid)
     meta = tracker.get_meta(options={'description', 'level', 'mails', 'org', 'org_name', 'filters', 'sparkline', 'tags',
                                      'filter_duplicate_notification',
-                                     'user', 'webhooks', 'nb_objs', 'years'})
+                                     'user', 'webhooks', 'nb_objs', 'objs_stats', 'years'})
 
     if meta['type'] == 'yara':
         yara_rule_content = Tracker.get_yara_rule_content(meta['tracked'])
@@ -216,7 +216,14 @@ def show_tracker():
     if date_from:
         date_from, date_to = Date.sanitise_daterange(date_from, date_to)
         objs = tracker.get_objs_by_daterange(date_from, date_to, filter_obj_types)
-        meta['objs'] = ail_objects.get_objects_meta(objs, options={'last_full_date', 'pdf'}, flask_context=True)
+        meta['objs'] = []
+        options = {'last_full_date', 'pdf'}
+        for obj_gid in objs:
+            obj_type, obj_subtype, obj_id = obj_gid.split(':', 2)
+            obj_meta = ail_objects.get_object_meta(obj_type, obj_subtype, obj_id, options=options, flask_context=True)
+            obj_meta['tracker_status'] = tracker.get_obj_status(obj_gid)
+            obj_meta['gid'] = obj_gid
+            meta['objs'].append(obj_meta)
     else:
         date_from = ''
         date_to = ''
@@ -501,6 +508,77 @@ def tracker_object_remove():
         else:
             return redirect(url_for('hunters.show_tracker', uuid=tracker_uuid))
 
+@hunters.route('/tracker/object/status/done', methods=['GET'])
+@login_required
+@login_user_no_api
+def tracker_object_status_done():
+    user_id = current_user.get_user_id()
+    user_org = current_user.get_org()
+    user_role = current_user.get_role()
+    tracker_uuid = request.args.get('uuid')
+    object_global_id = request.args.get('gid')
+
+    res = Tracker.api_tracker_object_status_done({'uuid': tracker_uuid, 'gid': object_global_id}, user_org, user_id, user_role)
+    if res[1] != 200:
+        return create_json_response(res[0], res[1])
+    else:
+        if request.referrer:
+            return redirect(request.referrer)
+        else:
+            return redirect(url_for('hunters.show_tracker', uuid=tracker_uuid))
+
+@hunters.route('/tracker/object/status/unread', methods=['GET'])
+@login_required
+@login_user_no_api
+def tracker_object_status_unread():
+    user_id = current_user.get_user_id()
+    user_org = current_user.get_org()
+    user_role = current_user.get_role()
+    tracker_uuid = request.args.get('uuid')
+    object_global_id = request.args.get('gid')
+
+    res = Tracker.api_tracker_object_status_unread({'uuid': tracker_uuid, 'gid': object_global_id}, user_org, user_id, user_role)
+    if res[1] != 200:
+        return create_json_response(res[0], res[1])
+    else:
+        if request.referrer:
+            return redirect(request.referrer)
+        else:
+            return redirect(url_for('hunters.show_tracker', uuid=tracker_uuid))
+
+
+@hunters.route('/tracker/object/status/reject', methods=['GET'])
+@login_required
+@login_user_no_api
+def tracker_object_status_reject():
+    user_id = current_user.get_user_id()
+    user_org = current_user.get_org()
+    user_role = current_user.get_role()
+    tracker_uuid = request.args.get('uuid')
+    object_global_id = request.args.get('gid')
+
+    res = Tracker.api_tracker_object_status_reject({'uuid': tracker_uuid, 'gid': object_global_id}, user_org, user_id, user_role)
+    if res[1] != 200:
+        return create_json_response(res[0], res[1])
+    else:
+        if request.referrer:
+            return redirect(request.referrer)
+        else:
+            return redirect(url_for('hunters.show_tracker', uuid=tracker_uuid))
+
+@hunters.route('/tracker/object/status/read', methods=['POST'])
+@login_required
+@login_user_no_api
+def tracker_object_status_read():
+    user_id = current_user.get_user_id()
+    user_org = current_user.get_org()
+    user_role = current_user.get_role()
+    tracker_uuid = request.args.get('uuid')
+    object_global_id = request.args.get('gid')
+
+    res = Tracker.api_tracker_object_status_read({'uuid': tracker_uuid, 'gid': object_global_id}, user_org, user_id, user_role)
+    return create_json_response(res[0], res[1])
+
 
 @hunters.route('/tracker/objects', methods=['GET'])
 @login_required