porting sqlite operations to APSW along with unittests for the same

nettacker/config.py

@@ -82,7 +82,11 @@ class DbConfig(ConfigBase):
     For sqlite database:
         fill the name of the DB as sqlite,
         DATABASE as the name of the db user wants
-        other details can be left empty
+        Set the journal_mode (default="WAL") and
+        synchronous_mode (default="NORMAL"). Rest
+        of the fields can be left emptyAdd commentMore actions
+        This is the default database:
+        str(CWD / ".data/nettacker.db")
     For mysql users:
         fill the ENGINE name of the DB as mysql
         NAME as the name of the database you want to create
@@ -104,6 +108,8 @@ class DbConfig(ConfigBase):
     username = ""
     password = ""
     ssl_mode = "disable"
+    journal_mode = "WAL"
+    synchronous_mode = "NORMAL"
 
 
 class PathConfig:
@@ -151,6 +157,8 @@ class DefaultSettings(ConfigBase):
         random_chars=generate_random_token(10),
     )
     retries = 1
+    max_retries = 3
+    retry_delay = 0.1
     scan_ip_range = False
     scan_subdomains = False
     selected_modules = None

nettacker/core/app.py

@@ -158,9 +158,7 @@ def expand_targets(self, scan_id):
 
             for target in copy.deepcopy(self.arguments.targets):
                 for row in find_events(target, "subdomain_scan", scan_id):
-                    for sub_domain in json.loads(row.json_event)["response"]["conditions_results"][
-                        "content"
-                    ]:
+                    for sub_domain in json.loads(row)["response"]["conditions_results"]["content"]:
                         if sub_domain not in self.arguments.targets:
                             self.arguments.targets.append(sub_domain)
         # icmp_scan

nettacker/core/graph.py

@@ -84,7 +84,7 @@ def build_text_table(events):
     table_headers = ["date", "target", "module_name", "port", "logs"]
     _table.add_rows([table_headers])
     for event in events:
-        log = merge_logs_to_list(json.loads(event["json_event"]), [])
+        log = merge_logs_to_list(event, [])
         _table.add_rows(
             [
                 table_headers,
@@ -157,15 +157,15 @@ def create_report(options, scan_id):
         )
         index = 1
         for event in all_scan_logs:
-            log_list = merge_logs_to_list(json.loads(event["json_event"]), [])
+            log_list = merge_logs_to_list(event, [])
             html_table_content += log_data.table_items.format(
                 event["date"],
                 event["target"],
                 event["module_name"],
                 event["port"],
                 "<br>".join(log_list) if log_list else "Detected",  # event["event"], #log
                 index,
-                html.escape(event["json_event"]),
+                html.escape(json.dumps(event)),
             )
             index += 1
         html_table_content += (

nettacker/core/lib/base.py

@@ -52,7 +52,7 @@ def get_dependent_results_from_database(self, target, module_name, scan_id, even
             while True:
                 event = find_temp_events(target, module_name, scan_id, event_name)
                 if event:
-                    events.append(json.loads(event.event)["response"]["conditions_results"])
+                    events.append(event["response"]["conditions_results"])
                     break
                 time.sleep(0.1)
         return events

nettacker/core/module.py

@@ -78,7 +78,7 @@ def load(self):
         if not self.skip_service_discovery and self.module_name not in self.ignored_core_modules:
             services = {}
             for service in find_events(self.target, "port_scan", self.scan_id):
-                service_event = json.loads(service.json_event)
+                service_event = json.loads(service)
                 port = service_event["port"]
                 protocols = service_event["response"]["conditions_results"].keys()
                 for protocol in protocols:

nettacker/core/utils/common.py

@@ -3,6 +3,7 @@
 import datetime
 import hashlib
 import importlib
+import json
 import math
 import multiprocessing
 import random
@@ -32,6 +33,10 @@ def replace_dependent_response(log, response_dependent):
 
 def merge_logs_to_list(result, log_list=[]):
     if isinstance(result, dict):
+        # Doesn't hurt normal operations
+        if "json_event" in list(result.keys()):
+            if not isinstance(result["json_event"], dict):
+                result["json_event"] = json.loads(result["json_event"])
         for i in result:
             if "log" == i:
                 log_list.append(result["log"])