chg: [crawler] add function to recrawl onion by month or to crawl all onion

Author: Terrtia

bin/lib/crawlers.py

@@ -1895,6 +1895,25 @@ def create_task(url, depth=1, har=True, screenshot=True, header=None, cookiejar=
                             external=external, new_task=new_task)
     return task_uuid
 
+def recrawl_domain(domain_id):
+    domain = Domains.Domain(domain_id)
+    parent = domain.get_parent()
+    if not parent:
+        parent = 'manual'
+    task_uuid = create_task(domain.id, parent=parent, priority=0, new_task=True, har=D_HAR, screenshot=D_SCREENSHOT)
+    if task_uuid:
+        print(task_uuid, domain.id, parent)
+
+def recrawl_onion_domains(date_month=None, all_onions_up=False):  # TODO RENAME ME
+    if all_onions_up:
+        to_crawl = Domains.get_domains_up_by_type('onion')
+    else:
+        if not date_month:
+            date_month = Date.get_previous_month_date()
+        to_crawl = set(Domains.get_domains_by_month(date_month, ['onion']))
+    for onion in to_crawl:
+        recrawl_domain(onion)
+
 ## -- CRAWLER TASK -- ##
 
 #### CRAWLER TASK API ####
@@ -2439,6 +2458,7 @@ def change_onion_filter_unknown_state(new_state):
 load_blacklist()
 
 # if __name__ == '__main__':
+#     recrawl_onion_domains(date_month='202502', all_onions_up=False)
 #     delete_captures()
 #
 #     item_id = 'crawled/2023/02/20/data.gz'

bin/lib/objects/Domains.py

@@ -609,6 +609,19 @@ def get_domains_dates_by_daterange(date_from, date_to, domain_types, up=True, do
                 date_domains[date] = list(domains)
     return date_domains
 
+def get_domains_by_month(date_month, domains_types, up=True, down=True):
+    start = f'{date_month}01'
+    end = Date.get_month_last_day(date_month)
+    if 'onion' in domains_types:
+        domains = get_domains_by_daterange(start, end, 'onion', up=up, down=down)
+    else:
+        domains = []
+    if 'web' in domains_types:
+        web = get_domains_by_daterange(start, end, 'web', up=up, down=down)
+        if web:
+            domains.extend(web)
+    return domains
+
 def get_domain_up_iterator():
     for domain_type in get_all_domains_types():
         for dom_id in get_domains_up_by_type(domain_type):

bin/packages/Date.py

@@ -8,7 +8,7 @@
 from dateutil.relativedelta import relativedelta
 
 def convert_date_str_to_datetime(date_str):
-    res =  datetime.date(int(date_str[0:4]), int(date_str[4:6]), int(date_str[6:8]))
+    res = datetime.date(int(date_str[0:4]), int(date_str[4:6]), int(date_str[6:8]))
     return res
 
 def get_full_month_str(date_from, date_to):
@@ -281,6 +281,14 @@ def get_previous_month_date():
     last_month = first - datetime.timedelta(days=1)
     return last_month.strftime("%Y%m%d")
 
+def get_month_last_day(date_month):
+    month = int(date_month[4:6]) % 12 + 1
+    return (datetime.date(int(date_month[0:4]), month, 1) - datetime.timedelta(days=1)).strftime("%Y%m%d")
+
+def get_current_month():
+    dt = datetime.date.today()
+    return dt.strftime("%Y%m")
+
 def get_current_year():
     dt = datetime.date.today()
     return dt.strftime("%Y")