diff --git a/custom_jinja_filters.py b/custom_jinja_filters.py index 654d694ff53..83cde0013c0 100644 --- a/custom_jinja_filters.py +++ b/custom_jinja_filters.py @@ -71,7 +71,7 @@ def clean_stix_data(data): def get_citations(data): """Given a description, find all of the citations.""" - p = re.compile("\(Citation: (.*?)\)") + p = re.compile(r"\(Citation: (.*?)\)") return p.findall(data) @@ -92,7 +92,7 @@ def get_html_citation(citations, citation_name): if citation: ref_number = None description = citation.get("description") - + if citation.get("number"): ref_number = citation["number"] else: @@ -101,10 +101,19 @@ def get_html_citation(citations, citation_name): citation["number"] = ref_number if not citation.get("url"): - reference_html = reference_marker_template_no_url.format(ref_number, ref_number, citation_name, description, ref_number) + reference_html = reference_marker_template_no_url.format( + ref_number, ref_number, citation_name, description, ref_number + ) else: reference_html = reference_marker_template.format( - ref_number, ref_number, description, citation_name, citation["url"], ref_number - 1, ref_number - 1, ref_number + ref_number, + ref_number, + description, + citation_name, + citation["url"], + ref_number - 1, + ref_number - 1, + ref_number, ) return reference_html @@ -158,7 +167,7 @@ def stixToHTML(data, citations, firstParagraphOnly, convert): citations (optional, object), if not None, add citation markers to the data. firstParagraphOnly (optional, boolean), if true, only return the first paragraph of the data in question. """ - if (convert): + if convert: # Replace data from markdown format data = markdown.markdown(data) diff --git a/modules/__init__.py b/modules/__init__.py index eda35e27112..e9636e7878a 100644 --- a/modules/__init__.py +++ b/modules/__init__.py @@ -14,7 +14,9 @@ def sort_menu_by_priority(): def sort_run_ptr_by_priority(): global run_ptr run_ptr = sorted(run_ptr, key=lambda k: k["priority"]) - print(f"Building website using the following modules in this order: {[pointer_info['module_name'] for pointer_info in run_ptr]}") + print( + f"Building website using the following modules in this order: {[pointer_info['module_name'] for pointer_info in run_ptr]}" + ) def check_redirections(redirections_list): diff --git a/modules/assets/__init__.py b/modules/assets/__init__.py index f39002b53e5..64f6ab8ef60 100644 --- a/modules/assets/__init__.py +++ b/modules/assets/__init__.py @@ -5,5 +5,6 @@ def get_priority(): return assets_config.priority + def run_module(): return (assets.generate_assets(), assets_config.module_name) diff --git a/modules/assets/assets.py b/modules/assets/assets.py index 25c166e201e..26f45360d10 100644 --- a/modules/assets/assets.py +++ b/modules/assets/assets.py @@ -51,18 +51,14 @@ def generate_markdown_files(): data = {} notes = util.relationshipgetters.get_objects_using_notes() - side_menu_data = util.buildhelpers.get_side_menu_data( - "Assets", "/assets/", asset_list_no_deprecated_revoked - ) + side_menu_data = util.buildhelpers.get_side_menu_data("Assets", "/assets/", asset_list_no_deprecated_revoked) data["side_menu_data"] = side_menu_data data["assets_table"] = get_assets_table_data(asset_list_no_deprecated_revoked) data["assets_list_len"] = str(len(asset_list_no_deprecated_revoked)) subs = assets_config.asset_index_md + json.dumps(data) - with open( - os.path.join(assets_config.asset_markdown_path, "overview.md"), "w", encoding="utf8" - ) as md_file: + with open(os.path.join(assets_config.asset_markdown_path, "overview.md"), "w", encoding="utf8") as md_file: md_file.write(subs) # Create the markdown for assets @@ -77,7 +73,8 @@ def generate_asset_md(asset, side_menu_data, notes): attack_id = util.buildhelpers.get_attack_id(asset) - if not attack_id: return + if not attack_id: + return data = {} data["attack_id"] = attack_id @@ -178,7 +175,8 @@ def get_assets_table_data(asset_list): assets_table_data = [] for asset in asset_list: attack_id = util.buildhelpers.get_attack_id(asset) - if not attack_id: continue + if not attack_id: + continue domain_list = util.buildhelpers.get_domain_name(asset) row = { @@ -187,7 +185,7 @@ def get_assets_table_data(asset_list): } for domain_idx in range(len(domain_list)): - domain_list[domain_idx] = domain_list[domain_idx].replace('-attack','') + domain_list[domain_idx] = domain_list[domain_idx].replace("-attack", "") if domain_list[domain_idx] == "ics": domain_list[domain_idx] = domain_list[domain_idx].upper() else: @@ -206,12 +204,13 @@ def get_assets_table_data(asset_list): def get_related_asset_data(related_assets): - if not related_assets: return [] + if not related_assets: + return [] related_asset_data = [] for related_asset in related_assets: row = { - "name": related_asset["name"], # required + "name": related_asset["name"], # required } if related_asset.get("related_asset_sectors"): related_asset["related_asset_sectors"].sort() @@ -224,7 +223,7 @@ def get_related_asset_data(related_assets): def get_techniques_targeting_asset_data(asset, reference_list): """Given an asset and its reference list, get the techniques targeting the asset. - Check the reference list for citations, if not found in list, add it. + Check the reference list for citations, if not found in list, add it. """ technique_list = {} techniques_targeting_assets = util.relationshipgetters.get_techniques_targeting_assets() diff --git a/modules/assets/assets_config.py b/modules/assets/assets_config.py index deafdb527d0..49c583c4a21 100644 --- a/modules/assets/assets_config.py +++ b/modules/assets/assets_config.py @@ -7,14 +7,10 @@ asset_markdown_path = "content/pages/assets/" # String template for asset index page -asset_index_md = ( - "Title: Asset overview\nTemplate: assets/assets-index\nsave_as: assets/index.html\ndata: " -) +asset_index_md = "Title: Asset overview\nTemplate: assets/assets-index\nsave_as: assets/index.html\ndata: " # String template for asset page -asset_md = Template( - "Title: ${name}\nTemplate: assets/asset\nsave_as: assets/${attack_id}/index.html\ndata: " -) +asset_md = Template("Title: ${name}\nTemplate: assets/asset\nsave_as: assets/${attack_id}/index.html\ndata: ") # Path for templates assets_templates_path = "modules/assets/templates/" diff --git a/modules/benefactors/__init__.py b/modules/benefactors/__init__.py index 68eb55f3e77..8fd0ea8cb93 100644 --- a/modules/benefactors/__init__.py +++ b/modules/benefactors/__init__.py @@ -17,5 +17,6 @@ def get_menu(): "children": [], } + def run_module(): return (benefactors.generate_benefactors(), benefactors_config.module_name) diff --git a/modules/benefactors/benefactors.py b/modules/benefactors/benefactors.py index 4861ab2fdbe..ac6045a0221 100644 --- a/modules/benefactors/benefactors.py +++ b/modules/benefactors/benefactors.py @@ -24,5 +24,7 @@ def generate_benefactors(): benefactors_md = benefactors_config.benefactors_md # write markdown to file - with open(os.path.join(benefactors_config.benefactors_markdown_path, "benefactors.md"), "w", encoding="utf8") as md_file: + with open( + os.path.join(benefactors_config.benefactors_markdown_path, "benefactors.md"), "w", encoding="utf8" + ) as md_file: md_file.write(benefactors_md) diff --git a/modules/benefactors/benefactors_config.py b/modules/benefactors/benefactors_config.py index 29ef1259880..fac102ce451 100644 --- a/modules/benefactors/benefactors_config.py +++ b/modules/benefactors/benefactors_config.py @@ -6,7 +6,10 @@ # String template for benefactors index page benefactors_md = ( - "Title: Benefactors\n" "Template: benefactors/benefactors\n" "save_as: resources/engage-with-attack/benefactors/index.html\n" "data: " + "Title: Benefactors\n" + "Template: benefactors/benefactors\n" + "save_as: resources/engage-with-attack/benefactors/index.html\n" + "data: " ) benefactors_templates_path = "modules/benefactors/templates" diff --git a/modules/campaigns/__init__.py b/modules/campaigns/__init__.py index 99ff4eaa234..cf52cc31ba6 100644 --- a/modules/campaigns/__init__.py +++ b/modules/campaigns/__init__.py @@ -5,5 +5,6 @@ def get_priority(): return campaigns_config.priority + def run_module(): return (campaigns.generate_campaigns(), campaigns_config.module_name) diff --git a/modules/campaigns/campaigns.py b/modules/campaigns/campaigns.py index eafbbadfcfd..4e61f930bf0 100644 --- a/modules/campaigns/campaigns.py +++ b/modules/campaigns/campaigns.py @@ -306,6 +306,7 @@ def get_software_table_data(campaign, reference_list): software_data = sorted(software_data, key=lambda k: k["name"].lower()) return software_data + def generate_sidebar_campaigns(side_menu_data): """Responsible for generating the sidebar for the campaigns pages.""" logger.info("Generating campaigns sidebar") @@ -316,5 +317,7 @@ def generate_sidebar_campaigns(side_menu_data): sidebar_campaigns_md = campaigns_config.sidebar_campaigns_md + json.dumps(data) # write markdown to file - with open(os.path.join(campaigns_config.campaign_markdown_path, "sidebar_campaigns.md"), "w", encoding="utf8") as md_file: + with open( + os.path.join(campaigns_config.campaign_markdown_path, "sidebar_campaigns.md"), "w", encoding="utf8" + ) as md_file: md_file.write(sidebar_campaigns_md) diff --git a/modules/campaigns/campaigns_config.py b/modules/campaigns/campaigns_config.py index 3eae07cc697..5d908dd6400 100644 --- a/modules/campaigns/campaigns_config.py +++ b/modules/campaigns/campaigns_config.py @@ -8,12 +8,12 @@ # String template for campaign index page campaign_index_md = ( - "Title: Campaign overview\n" "Template: campaigns/campaigns-index\n" "save_as: campaigns/index.html\n" "data: " + "Title: Campaign overview\nTemplate: campaigns/campaigns-index\nsave_as: campaigns/index.html\ndata: " ) # String template for campaign page campaign_md = Template( - "Title: ${name}\n" "Template: campaigns/campaign\n" "save_as: campaigns/${attack_id}/index.html\n" "data: " + "Title: ${name}\nTemplate: campaigns/campaign\nsave_as: campaigns/${attack_id}/index.html\ndata: " ) # Path for templates @@ -26,4 +26,4 @@ "Template: general/sidebar-template \n" "save_as: campaigns/sidebar-campaigns/index.html\n" "data: " -) \ No newline at end of file +) diff --git a/modules/datasources/__init__.py b/modules/datasources/__init__.py index 56b61bcbdd9..8d40ac5f5ab 100644 --- a/modules/datasources/__init__.py +++ b/modules/datasources/__init__.py @@ -16,13 +16,23 @@ def get_menu(): "priority": datasources_config.priority, "children": [ {"display_name": "Data Sources", "url": "/datasources", "external_link": False, "children": []}, - {"display_name": "Mitigations", "url": "/mitigations/", "external_link": False, "children": [ - {"display_name": "Enterprise", "url": "/mitigations/enterprise/", "external_link": False, "children": []}, - {"display_name": "Mobile", "url": "/mitigations/mobile/", "external_link": False, "children": []}, - {"display_name": "ICS", "url": "/mitigations/ics/", "external_link": False, "children": []}, - ]}, - {"display_name": "Assets", "url": "/assets", "external_link": False, "children": [] } - ] + { + "display_name": "Mitigations", + "url": "/mitigations/", + "external_link": False, + "children": [ + { + "display_name": "Enterprise", + "url": "/mitigations/enterprise/", + "external_link": False, + "children": [], + }, + {"display_name": "Mobile", "url": "/mitigations/mobile/", "external_link": False, "children": []}, + {"display_name": "ICS", "url": "/mitigations/ics/", "external_link": False, "children": []}, + ], + }, + {"display_name": "Assets", "url": "/assets", "external_link": False, "children": []}, + ], } diff --git a/modules/datasources/datasources.py b/modules/datasources/datasources.py index 663a2a41f80..20f67914b74 100644 --- a/modules/datasources/datasources.py +++ b/modules/datasources/datasources.py @@ -229,7 +229,7 @@ def get_datasources_table_data(datasource_list): row["id"] = attack_id for domain_idx in range(len(domain_list)): - domain_list[domain_idx] = domain_list[domain_idx].replace('-attack','') + domain_list[domain_idx] = domain_list[domain_idx].replace("-attack", "") if domain_list[domain_idx] == "ics": domain_list[domain_idx] = domain_list[domain_idx].upper() else: @@ -322,21 +322,22 @@ def get_datacomponents_data(datasource, reference_list): def get_domains_of_datacomponent(datacomponent, techniques_detected_by_datacomponent, technique_to_domain): - """Retrives domains of given data component""" - domains_of_datacomponent = [] + """Retrives domains of given data component""" + domains_of_datacomponent = [] - # get data components to techniques mapping to find domains - techniques_of_datacomp = techniques_detected_by_datacomponent.get(datacomponent["id"]) - if techniques_of_datacomp: - technique_list = {} - for technique_rel in techniques_of_datacomp: - attack_id = util.buildhelpers.get_attack_id(technique_rel["object"]) - if attack_id: - domain = technique_to_domain[attack_id].split("-")[0] - if not domain in domains_of_datacomponent: - domains_of_datacomponent.append(domain) + # get data components to techniques mapping to find domains + techniques_of_datacomp = techniques_detected_by_datacomponent.get(datacomponent["id"]) + if techniques_of_datacomp: + technique_list = {} + for technique_rel in techniques_of_datacomp: + attack_id = util.buildhelpers.get_attack_id(technique_rel["object"]) + if attack_id: + domain = technique_to_domain[attack_id].split("-")[0] + if not domain in domains_of_datacomponent: + domains_of_datacomponent.append(domain) + + return domains_of_datacomponent - return domains_of_datacomponent def get_datasources_domain(datasource): """Responsible for generating the list of domains for the datasources and datacomponents.""" @@ -357,10 +358,12 @@ def get_datasources_domain(datasource): # get data component detections techniques_of_datacomp = techniques_detected_by_datacomponent.get(datacomponent["id"]) if techniques_of_datacomp: - domains_of_datacomponent = get_domains_of_datacomponent(datacomponent, techniques_detected_by_datacomponent, technique_to_domain) + domains_of_datacomponent = get_domains_of_datacomponent( + datacomponent, techniques_detected_by_datacomponent, technique_to_domain + ) # Add missing domains to data source for domain in domains_of_datacomponent: if not domain in domains_of_datasource: domains_of_datasource.append(domain) - return domains_of_datasource \ No newline at end of file + return domains_of_datasource diff --git a/modules/datasources/datasources_config.py b/modules/datasources/datasources_config.py index 584af6d1eff..3d062782e77 100644 --- a/modules/datasources/datasources_config.py +++ b/modules/datasources/datasources_config.py @@ -11,16 +11,13 @@ # String template for data sources index page datasource_index_md = ( - "Title: Data Sources overview\n" - "Template: datasources/datasources-index\n" - "save_as: datasources/index.html\n" - "data: " + "Title: Data Sources overview\nTemplate: datasources/datasources-index\nsave_as: datasources/index.html\ndata: " ) # String template for data source page datasource_md = Template( - "Title: ${name}\n" "Template: datasources/datasource\n" "save_as: datasources/${attack_id}/index.html\n" "data: " + "Title: ${name}\nTemplate: datasources/datasource\nsave_as: datasources/${attack_id}/index.html\ndata: " ) # Path for templates -datasources_templates_path = "modules/datasources/templates/" \ No newline at end of file +datasources_templates_path = "modules/datasources/templates/" diff --git a/modules/groups/__init__.py b/modules/groups/__init__.py index abe5567a490..608f6037250 100644 --- a/modules/groups/__init__.py +++ b/modules/groups/__init__.py @@ -6,6 +6,7 @@ def get_priority(): return groups_config.priority + def get_menu(): return { "display_name": groups_config.module_tab_name, @@ -17,8 +18,9 @@ def get_menu(): {"display_name": "Groups", "url": "/groups", "external_link": False, "children": []}, {"display_name": "Software", "url": "/software", "external_link": False, "children": []}, {"display_name": "Campaigns", "url": "/campaigns", "external_link": False, "children": []}, - ] + ], } + def run_module(): return (groups.generate_groups(), groups_config.module_name) diff --git a/modules/groups/groups_config.py b/modules/groups/groups_config.py index 20a7da88a4e..a096ebf0367 100644 --- a/modules/groups/groups_config.py +++ b/modules/groups/groups_config.py @@ -8,10 +8,10 @@ group_markdown_path = "content/pages/groups/" # String template for group index page -group_index_md = "Title: Group overview\n" "Template: groups/groups-index\n" "save_as: groups/index.html\n" "data: " +group_index_md = "Title: Group overview\nTemplate: groups/groups-index\nsave_as: groups/index.html\ndata: " # String template for group page -group_md = Template("Title: ${name}\n" "Template: groups/group\n" "save_as: groups/${attack_id}/index.html\n" "data: ") +group_md = Template("Title: ${name}\nTemplate: groups/group\nsave_as: groups/${attack_id}/index.html\ndata: ") # Path for templates groups_templates_path = "modules/groups/templates/" @@ -19,8 +19,5 @@ groups_redirection_location = "modules/groups/groups_redirections.json" sidebar_groups_md = ( - "Title: Groups Sidebar\n" - "Template: general/sidebar-template \n" - "save_as: groups/sidebar-groups/index.html\n" - "data: " -) \ No newline at end of file + "Title: Groups Sidebar\nTemplate: general/sidebar-template \nsave_as: groups/sidebar-groups/index.html\ndata: " +) diff --git a/modules/matrices/matrices.py b/modules/matrices/matrices.py index 22a3ab254bf..741f17671a7 100644 --- a/modules/matrices/matrices.py +++ b/modules/matrices/matrices.py @@ -260,6 +260,7 @@ def transform_tactic(tactic_id): return data, has_subtechniques, tour_technique + def generate_sidebar_matrices(side_menu_data): """Responsible for generating the sidebar for the matrices pages.""" logger.info("Generating matrices sidebar") @@ -270,5 +271,7 @@ def generate_sidebar_matrices(side_menu_data): sidebar_matrices_md = matrices_config.sidebar_matrices_md + json.dumps(data) # write markdown to file - with open(os.path.join(matrices_config.matrix_markdown_path, "sidebar_matrices.md"), "w", encoding="utf8") as md_file: + with open( + os.path.join(matrices_config.matrix_markdown_path, "sidebar_matrices.md"), "w", encoding="utf8" + ) as md_file: md_file.write(sidebar_matrices_md) diff --git a/modules/matrices/matrices_config.py b/modules/matrices/matrices_config.py index be228ef91bd..0a0408519e4 100644 --- a/modules/matrices/matrices_config.py +++ b/modules/matrices/matrices_config.py @@ -18,9 +18,7 @@ ) # String template for main domain matrices -matrix_md = Template( - "Title: Matrix-${domain}\n" "Template: matrices/matrix\n" "save_as: matrices/${path}/index.html\n" "data: " -) +matrix_md = Template("Title: Matrix-${domain}\nTemplate: matrices/matrix\nsave_as: matrices/${path}/index.html\ndata: ") # String template for platform matrices platform_md = Template( diff --git a/modules/mitigations/mitigations.py b/modules/mitigations/mitigations.py index e5bab98ef11..f89e2ace629 100644 --- a/modules/mitigations/mitigations.py +++ b/modules/mitigations/mitigations.py @@ -229,6 +229,7 @@ def get_techniques_addressed_data(mitigation, reference_list): ) return technique_data + def generate_sidebar_mitigations(side_nav_data): """Responsible for generating the sidebar for the mitigations pages.""" logger.info("Generating mitigations sidebar") @@ -239,5 +240,7 @@ def generate_sidebar_mitigations(side_nav_data): sidebar_mitigations_md = mitigations_config.sidebar_mitigations_md + json.dumps(data) # write markdown to file - with open(os.path.join(mitigations_config.mitigation_markdown_path, "sidebar_mitigations.md"), "w", encoding="utf8") as md_file: + with open( + os.path.join(mitigations_config.mitigation_markdown_path, "sidebar_mitigations.md"), "w", encoding="utf8" + ) as md_file: md_file.write(sidebar_mitigations_md) diff --git a/modules/mitigations/mitigations_config.py b/modules/mitigations/mitigations_config.py index f60ea69f4b4..1ba778f6bd4 100644 --- a/modules/mitigations/mitigations_config.py +++ b/modules/mitigations/mitigations_config.py @@ -27,10 +27,7 @@ # String template for all mitigations mitigation_md = Template( - "Title: ${name}-${domain}\n" - "Template: mitigations/mitigation\n" - "save_as: mitigations/${attack_id}/index.html\n" - "data: " + "Title: ${name}-${domain}\nTemplate: mitigations/mitigation\nsave_as: mitigations/${attack_id}/index.html\ndata: " ) sidebar_mitigations_md = ( @@ -38,4 +35,4 @@ "Template: general/sidebar-template \n" "save_as: mitigations/sidebar-mitigations/index.html\n" "data: " -) \ No newline at end of file +) diff --git a/modules/random_page/random_page.py b/modules/random_page/random_page.py index 604e7e65e17..09e73557439 100644 --- a/modules/random_page/random_page.py +++ b/modules/random_page/random_page.py @@ -16,7 +16,7 @@ def generate_json(): "groups": "Group", "software": "Software", "campaigns": "Campaign", - "assets": "Asset" + "assets": "Asset", } routes = {} diff --git a/modules/resources/__init__.py b/modules/resources/__init__.py index e1bd244cf5e..33477ec9925 100644 --- a/modules/resources/__init__.py +++ b/modules/resources/__init__.py @@ -27,12 +27,7 @@ def get_menu(): "external_link": False, "children": [], }, - { - "display_name": "ATT&CKcon", - "url": "/resources/attackcon/", - "external_link": False, - "children": [] - }, + {"display_name": "ATT&CKcon", "url": "/resources/attackcon/", "external_link": False, "children": []}, { "display_name": "ATT&CK Data & Tools", "url": "/resources/attack-data-and-tools/", @@ -63,7 +58,7 @@ def get_menu(): "url": "/resources/legal-and-branding/", "external_link": False, "children": [], - } + }, ], } diff --git a/modules/resources/resources.py b/modules/resources/resources.py index f79d78488c3..c287dbcbd97 100644 --- a/modules/resources/resources.py +++ b/modules/resources/resources.py @@ -15,6 +15,7 @@ import urllib.parse + def generate_resources(): """Responsible for generating the resources pages.""" logger.info("Generating Resources") @@ -62,6 +63,7 @@ def generate_resources(): generate_use_case_page() generate_sidebar_resources() + def copy_docs(module_docs_path): """Move module specific docs into the website's content directory for pelican.""" logger.info("Copying files to docs directory") @@ -79,17 +81,20 @@ def copy_docs(module_docs_path): else: shutil.copyfile(os.path.join(module_docs_path, doc), os.path.join(site_config.docs_dir, doc)) + def extract_video_id(url): - if '=' in url and '&' in url: - start = url.find('=') + 1 - end = url.find('&', start) + if "=" in url and "&" in url: + start = url.find("=") + 1 + end = url.find("&", start) return url[start:end] return url + def extract_playlist_id(url): parsed_url = urllib.parse.urlparse(url) query_params = urllib.parse.parse_qs(parsed_url.query) - return query_params.get('list', [None])[0] + return query_params.get("list", [None])[0] + def generate_training_pages(): """Responsible for generating the markdown pages of the training pages.""" @@ -104,13 +109,22 @@ def generate_training_pages(): if "lessons" in trainings[training_main][module]: if trainings[training_main][module]["is_youtube"]: # if videos are in a playlist - if "playlist" in trainings[training_main][module] and trainings[training_main][module]["playlist"] == True: - trainings[training_main][module]["first_video_id"] = extract_video_id(trainings[training_main][module]["lessons"][0]["youtube"]) - trainings[training_main][module]["playlist_id"] = extract_playlist_id(trainings[training_main][module]["lessons"][0]["youtube"]) + if ( + "playlist" in trainings[training_main][module] + and trainings[training_main][module]["playlist"] == True + ): + trainings[training_main][module]["first_video_id"] = extract_video_id( + trainings[training_main][module]["lessons"][0]["youtube"] + ) + trainings[training_main][module]["playlist_id"] = extract_playlist_id( + trainings[training_main][module]["lessons"][0]["youtube"] + ) # if videos are not in a playlist else: for lesson_idx in range(len(trainings[training_main][module]["lessons"])): - trainings[training_main][module]["lessons"][lesson_idx]["video_id"] = extract_video_id(trainings[training_main][module]["lessons"][lesson_idx]["youtube"]) + trainings[training_main][module]["lessons"][lesson_idx]["video_id"] = extract_video_id( + trainings[training_main][module]["lessons"][lesson_idx]["youtube"] + ) # Define a dictionary of training pages and their corresponding markdown templates training_pages = { @@ -128,9 +142,12 @@ def generate_training_pages(): # Generate markdown for each training page and write it to a file for page_name, page_template in training_pages.items(): page_content = page_template + json.dumps(trainings) - with open(os.path.join(site_config.resources_markdown_path, f"{page_name}.md"), "w", encoding="utf8") as md_file: + with open( + os.path.join(site_config.resources_markdown_path, f"{page_name}.md"), "w", encoding="utf8" + ) as md_file: md_file.write(page_content) + def generate_brand_page(): """Responsible for generating the markdown pages of the training pages.""" logger.info("Generating brand") @@ -212,6 +229,7 @@ def generate_faq_page(): with open(os.path.join(site_config.resources_markdown_path, "faq.md"), "w", encoding="utf8") as md_file: md_file.write(faq_content) + def generate_static_pages(): """Reads markdown files from the static pages directory and copies them into the markdown directory.""" logger.info("Generating static pages") @@ -219,7 +237,7 @@ def generate_static_pages(): if not [key["module_name"] for key in modules.run_ptr if key["module_name"] == "versions"]: util.buildhelpers.remove_element_from_sub_menu(resources_config.module_name, "Version History") - + # Below code used to get a list of all updates children updates_dict_list = {} updates_name = [] @@ -289,7 +307,7 @@ def generate_working_with_attack(): "techniques", "datasources", "campaigns", - "assets" + "assets", ] # Verify if directories exists @@ -342,7 +360,9 @@ def generate_sidebar_resources(): sidebar_resources_md = resources_config.sidebar_resources_md # write markdown to file - with open(os.path.join(site_config.resources_markdown_path, "sidebar_resources.md"), "w", encoding="utf8") as md_file: + with open( + os.path.join(site_config.resources_markdown_path, "sidebar_resources.md"), "w", encoding="utf8" + ) as md_file: md_file.write(sidebar_resources_md) @@ -360,7 +380,7 @@ def generate_contribute_page(): data = {} - data["contributors"] = [] + data["contributors"] = [] contributors_first_col = [] contributors_second_col = [] @@ -384,11 +404,10 @@ def generate_contribute_page(): subs = resources_config.contribute_md + json.dumps(data) # Open markdown file for the contribute page - with open( - os.path.join(site_config.resources_markdown_path, "contribute.md"), "w", encoding="utf8" - ) as md_file: + with open(os.path.join(site_config.resources_markdown_path, "contribute.md"), "w", encoding="utf8") as md_file: md_file.write(subs) + def generate_presentation_archive(): """Responsible for compiling resources json into resources markdown files for rendering on the HMTL.""" logger.info("Generating presentation archive") @@ -408,6 +427,7 @@ def generate_presentation_archive(): ) as md_file: md_file.write(resources_content) + def generate_use_case_page(): """Responsible for compiling use cases json into use cases markdown file for rendering on the HMTL.""" logger.info("Generating get started pages") @@ -424,7 +444,7 @@ def generate_use_case_page(): for i in range(len(use_case_data)): use_case_name.append(use_case_data[i]["title"]) title = "Title: " + use_case_data[i]["title"] + "\n" - name = use_case_data[i]["title"].lower().replace(' ','-').replace("&", "a") + name = use_case_data[i]["title"].lower().replace(" ", "-").replace("&", "a") template = "Template: resources/use-cases\n" use_case_path.append("/resources/get-started/" + name + "/") save_as = "save_as: resources/get-started/" + name + "/index.html\n" @@ -439,6 +459,6 @@ def generate_use_case_page(): use_case_list = use_case_dict_list["use_case_md"] for i in range(len(use_case_list)): use_case_content = use_case_list[i] + json.dumps(use_case_data[i]) - f_name = "use-case-" + use_case_data[i]["title"].lower().replace(' ','-') + ".md" + f_name = "use-case-" + use_case_data[i]["title"].lower().replace(" ", "-") + ".md" with open(os.path.join(site_config.resources_markdown_path, f_name), "w", encoding="utf8") as md_file: - md_file.write(use_case_content) \ No newline at end of file + md_file.write(use_case_content) diff --git a/modules/resources/resources_config.py b/modules/resources/resources_config.py index 8a46608b937..18ef48c3d9c 100644 --- a/modules/resources/resources_config.py +++ b/modules/resources/resources_config.py @@ -76,12 +76,7 @@ ) # FAQ md -faq_md = ( - "Title: Frequently Asked Questions\n" - "Template: general/faq-overview\n" - "save_as: resources/faq/index.html\n" - "data: " -) +faq_md = "Title: Frequently Asked Questions\nTemplate: general/faq-overview\nsave_as: resources/faq/index.html\ndata: " # Contribute md contribute_md = ( @@ -119,7 +114,5 @@ # Sidebar md sidebar_resources_md = ( - "Title: Resources Sidebar\n" - "Template: general/sidebar-resources \n" - "save_as: resources/sidebar-resources/index.html\n" + "Title: Resources Sidebar\nTemplate: general/sidebar-resources \nsave_as: resources/sidebar-resources/index.html\n" ) diff --git a/modules/site_config.py b/modules/site_config.py index 9bb53627140..d62e9e467a6 100644 --- a/modules/site_config.py +++ b/modules/site_config.py @@ -135,17 +135,9 @@ def set_subdirectory(subdirectory_str): # Redirect md string template redirect_md_index = Template( - "Title: ${title}\n" - "Template: general/redirect-index\n" - "RedirectLink: ${to}\n" - "save_as: ${from}/index.html" -) -redirect_md = Template( - "Title: ${title}\n" - "Template: general/redirect-index\n" - "RedirectLink: ${to}\n" - "save_as: ${from}" + "Title: ${title}\nTemplate: general/redirect-index\nRedirectLink: ${to}\nsave_as: ${from}/index.html" ) +redirect_md = Template("Title: ${title}\nTemplate: general/redirect-index\nRedirectLink: ${to}\nsave_as: ${from}") # Custom_alphabet used to sort list of dictionaries by domain name # depending on domain ordering @@ -174,10 +166,7 @@ def set_subdirectory(subdirectory_str): # domain: "enterprise", "mobile", "ics" # path: the path to the object, e.g "software/S1001" or "groups/G2021" layer_md = Template( - "Title: ${domain} Techniques\n" - "Template: general/json\n" - "save_as: ${path}/${attack_id}-${domain}-layer.json\n" - "json: " + "Title: ${domain} Techniques\nTemplate: general/json\nsave_as: ${path}/${attack_id}-${domain}-layer.json\njson: " ) layer_version = "4.5" navigator_version = "5.1.0" diff --git a/modules/software/__init__.py b/modules/software/__init__.py index 103553ec56d..e65709aeede 100644 --- a/modules/software/__init__.py +++ b/modules/software/__init__.py @@ -2,14 +2,17 @@ from . import software_config import json + def get_priority(): return software_config.priority + # TODO commented out to resolve infinite redirect loop when run locally. Needs further testing before code removal. # def get_redirections(): # with open(software_config.software_redirection_location , "r", encoding="utf8") as json_redirections: # return json.load(json_redirections) # return [] + def run_module(): return software.generate_software(), software_config.module_name diff --git a/modules/software/software_config.py b/modules/software/software_config.py index 304e3ca3a6f..e399491f585 100644 --- a/modules/software/software_config.py +++ b/modules/software/software_config.py @@ -10,16 +10,10 @@ software_templates_path = "modules/software/templates/" # String template for software index page -software_index_md = ("Title: Software overview\n" - "Template: software/software-index\n" - "save_as: software/index.html\n" - "data: ") +software_index_md = "Title: Software overview\nTemplate: software/software-index\nsave_as: software/index.html\ndata: " # String template for group page -software_md = Template("Title: ${name}\n" - "Template: software/software\n" - "save_as: software/${attack_id}/index.html\n" - "data: ") +software_md = Template("Title: ${name}\nTemplate: software/software\nsave_as: software/${attack_id}/index.html\ndata: ") software_redirection_location = "modules/software/software_redirections.json" @@ -28,4 +22,4 @@ "Template: general/sidebar-template \n" "save_as: software/sidebar-software/index.html\n" "data: " -) \ No newline at end of file +) diff --git a/modules/stixtests/linkbyidchecker.py b/modules/stixtests/linkbyidchecker.py index 04de5b8e15d..d66268fbb01 100644 --- a/modules/stixtests/linkbyidchecker.py +++ b/modules/stixtests/linkbyidchecker.py @@ -126,7 +126,7 @@ def linkbyid_check(): "Object with broken data": pretty_name, "Section found in": "Description", # sort of abusing "Unknown LinkById" here, but it works well enough - "Unknown LinkById": f"{{{{LinkById\|{attack_id_in_description}}}}}", + "Unknown LinkById": rf"{{{{LinkById|{attack_id_in_description}}}}}", } link_by_id_warnings.append(warning) diff --git a/modules/subdirectory/subdirectory.py b/modules/subdirectory/subdirectory.py index 0b42bcb7101..ad9c67d7d28 100644 --- a/modules/subdirectory/subdirectory.py +++ b/modules/subdirectory/subdirectory.py @@ -11,27 +11,7 @@ def generate_subdirectory(): replace() -allowed_in_link = "".join( - list( - map( - lambda s: s.strip(), - [ - " - ", - " ? ", - " \w ", - " \\ ", - " $ ", - " \. ", - " ! ", - " \* ", - " ' ", - " () ", - " / ", - ], - ) - ) -) - +allowed_in_link = r"-?\w\$\.!\*'()/" def replace_links(filepath): """In the given file, replace the in-site links to reference @@ -43,7 +23,7 @@ def replace_links(filepath): html_str = html.read() # subdirectory link format - dest_link_format = f"/{site_config.subdirectory}\g<1>" + dest_link_format = rf"/{site_config.subdirectory}\g<1>" def substitute(prefix, html_str): from_str = f"{prefix}=[\"']([{allowed_in_link}]+)[\"']" diff --git a/modules/tactics/__init__.py b/modules/tactics/__init__.py index 611812692c2..015f14636a8 100644 --- a/modules/tactics/__init__.py +++ b/modules/tactics/__init__.py @@ -21,6 +21,7 @@ def get_menu(): ], } + # TODO resolve infinite redirect loop when run locally. Needs further testing before code removal. def get_redirections(): with open(tactics_config.tactics_redirection_location, "r", encoding="utf8") as json_redirections: diff --git a/modules/tactics/tactics.py b/modules/tactics/tactics.py index 2008ddbd57b..2037c9bfc6d 100644 --- a/modules/tactics/tactics.py +++ b/modules/tactics/tactics.py @@ -190,6 +190,7 @@ def get_techniques_of_tactic(tactic, techniques): techniques_list = sorted(techniques_list, key=lambda k: k["name"].lower()) return techniques_list + def generate_sidebar_tactics(side_nav_data): """Responsible for generating the sidebar for the tactics pages.""" logger.info("Generating tactics sidebar") @@ -200,6 +201,7 @@ def generate_sidebar_tactics(side_nav_data): sidebar_tactics_md = tactics_config.sidebar_tactics_md + json.dumps(data) # write markdown to file - with open(os.path.join(tactics_config.tactics_markdown_path, "sidebar_tactics.md"), "w", encoding="utf8") as md_file: + with open( + os.path.join(tactics_config.tactics_markdown_path, "sidebar_tactics.md"), "w", encoding="utf8" + ) as md_file: md_file.write(sidebar_tactics_md) - diff --git a/modules/tactics/tactics_config.py b/modules/tactics/tactics_config.py index b8dd5e4fa71..99d3dd261c1 100644 --- a/modules/tactics/tactics_config.py +++ b/modules/tactics/tactics_config.py @@ -11,12 +11,12 @@ # String template for domains tactic_domain_md = Template( - "Title: Tactics\n" "Template: tactics/tactics-domain-index\n" "save_as: tactics/${domain}/index.html\n" "data: " + "Title: Tactics\nTemplate: tactics/tactics-domain-index\nsave_as: tactics/${domain}/index.html\ndata: " ) # String template for tactics tactic_md = Template( - "Title: ${name}-${domain}\n" "Template: tactics/tactic\n" "save_as: tactics/${attack_id}/index.html\n" "data: " + "Title: ${name}-${domain}\nTemplate: tactics/tactic\nsave_as: tactics/${attack_id}/index.html\ndata: " ) # Tactics overview md template @@ -30,8 +30,5 @@ tactics_redirection_location = "modules/tactics/tactics_redirections.json" sidebar_tactics_md = ( - "Title: Tactics Sidebar\n" - "Template: general/sidebar-template \n" - "save_as: tactics/sidebar-tactics/index.html\n" - "data: " -) \ No newline at end of file + "Title: Tactics Sidebar\nTemplate: general/sidebar-template \nsave_as: tactics/sidebar-tactics/index.html\ndata: " +) diff --git a/modules/techniques/techniques.py b/modules/techniques/techniques.py index 3d88ed39f3f..3f3fb7918be 100644 --- a/modules/techniques/techniques.py +++ b/modules/techniques/techniques.py @@ -259,7 +259,7 @@ def generate_data_for_md(technique_dict, technique, tactic_list, is_sub_techniqu if technique.get("x_mitre_system_requirements"): technique["x_mitre_system_requirements"].sort() technique_dict["sysreqs"] = ", ".join(technique["x_mitre_system_requirements"]) - technique_dict["sysreqs"] = re.sub("\.?\\n+", "; ", technique_dict["sysreqs"]) + technique_dict["sysreqs"] = re.sub(r"\.?\\n+", "; ", technique_dict["sysreqs"]) # Get permissions required if technique.get("x_mitre_permissions_required"): @@ -361,15 +361,14 @@ def get_mitigations_table_data(technique, reference_list): if not mitigation["object"].get("x_mitre_deprecated"): attack_id = util.buildhelpers.get_attack_id(mitigation["object"]) # Only add if mitigation attack id is found - if not attack_id: continue + if not attack_id: + continue row = {} row["mid"] = attack_id row["name"] = mitigation["object"]["name"] if mitigation["relationship"].get("description"): # Get filtered description - reference_list = util.buildhelpers.update_reference_list( - reference_list, mitigation["relationship"] - ) + reference_list = util.buildhelpers.update_reference_list(reference_list, mitigation["relationship"]) row["descr"] = mitigation["relationship"]["description"] mitigation_data.append(row) @@ -396,15 +395,14 @@ def get_assets_table_data(technique, reference_list): attack_id = util.buildhelpers.get_attack_id(asset["object"]) # Only add if attack id is found - if not attack_id: continue + if not attack_id: + continue row = {} row["id"] = attack_id row["name"] = asset["object"]["name"] if asset["relationship"].get("description"): # Get filtered description - reference_list = util.buildhelpers.update_reference_list( - reference_list, asset["relationship"] - ) + reference_list = util.buildhelpers.update_reference_list(reference_list, asset["relationship"]) row["descr"] = asset["relationship"]["description"] asset_data.append(row) @@ -436,7 +434,8 @@ def get_examples_table_data(technique, reference_list): attack_id = util.buildhelpers.get_attack_id(example["object"]) # Only add example data if the attack id is found - if not attack_id: continue + if not attack_id: + continue row = {} row["id"] = attack_id @@ -447,9 +446,7 @@ def get_examples_table_data(technique, reference_list): if example["relationship"].get("description"): # Get filtered description - reference_list = util.buildhelpers.update_reference_list( - reference_list, example["relationship"] - ) + reference_list = util.buildhelpers.update_reference_list(reference_list, example["relationship"]) row["descr"] = example["relationship"]["description"] example_data.append(row) @@ -544,7 +541,8 @@ def get_techniques_list(techniques): if not technique.get("revoked") and not technique.get("x_mitre_deprecated"): attack_id = util.buildhelpers.get_attack_id(technique) - if not attack_id: continue + if not attack_id: + continue technique_dict = {} technique_dict["id"] = attack_id @@ -649,6 +647,7 @@ def get_datasources_and_components_of_technique(technique, reference_list): return datasource_and_components, show_descriptions + def generate_sidebar_techniques(side_nav_data): """Responsible for generating the sidebar for the technique pages.""" logger.info("Generating technique sidebar") @@ -659,5 +658,7 @@ def generate_sidebar_techniques(side_nav_data): sidebar_techniques_md = techniques_config.sidebar_techniques_md + json.dumps(data) # write markdown to file - with open(os.path.join(techniques_config.techniques_markdown_path, "sidebar_techniques.md"), "w", encoding="utf8") as md_file: + with open( + os.path.join(techniques_config.techniques_markdown_path, "sidebar_techniques.md"), "w", encoding="utf8" + ) as md_file: md_file.write(sidebar_techniques_md) diff --git a/modules/techniques/techniques_config.py b/modules/techniques/techniques_config.py index 1679960fb7d..399c4f58fbd 100644 --- a/modules/techniques/techniques_config.py +++ b/modules/techniques/techniques_config.py @@ -11,18 +11,12 @@ # String template for all techniques technique_md = Template( - "Title: ${name}-${domain}\n" - "Template: techniques/technique\n" - "save_as: techniques/${attack_id}/index.html\n" - "data: " + "Title: ${name}-${domain}\nTemplate: techniques/technique\nsave_as: techniques/${attack_id}/index.html\ndata: " ) # String template for domains technique_domain_md = Template( - "Title: Techniques\n" - "Template: techniques/techniques-domain-index\n" - "save_as: techniques/${domain}/index.html\n" - "data: " + "Title: Techniques\nTemplate: techniques/techniques-domain-index\nsave_as: techniques/${domain}/index.html\ndata: " ) # Overview md template @@ -48,4 +42,4 @@ "Template: general/sidebar-template \n" "save_as: techniques/sidebar-techniques/index.html\n" "data: " -) \ No newline at end of file +) diff --git a/modules/tests/citationchecker.py b/modules/tests/citationchecker.py index d4870fa55a8..2509b19e7a7 100644 --- a/modules/tests/citationchecker.py +++ b/modules/tests/citationchecker.py @@ -7,7 +7,7 @@ from . import tests_config -potential_issues_list = ["\(Citation: ?[^)]+\)?"] +potential_issues_list = [r"\(Citation: ?[^)]+\)?"] def citations_check(): diff --git a/modules/tests/linkchecker.py b/modules/tests/linkchecker.py index 9cd1f339a31..9e2a7f1f8fb 100644 --- a/modules/tests/linkchecker.py +++ b/modules/tests/linkchecker.py @@ -1,61 +1,18 @@ import os import re -from pathlib import Path import requests -from loguru import logger -from modules import site_config import modules +from modules import site_config + from . import tests_config # STATIC PROPERTIES -# The spaces here are for readability with symbol characters. -# The strings are stripped of whitespace. -# These get compiled into a regex string - -allowed_in_link_with_external_links = "".join( - list( - map( - lambda s: s.strip(), - [ - " - ", - " ? ", - " \w ", - " \\ ", - " $ ", - " \. ", - " ! ", - " \* ", - " ' ", - " () ", - " / ", - " : ", - ], - ) - ) -) - -allowed_in_link = "".join( - list( - map( - lambda s: s.strip(), - [ - " - ", - " ? ", - " \w ", - " \\ ", - " $ ", - " \. ", - " ! ", - " \* ", - " ' ", - " () ", - " / ", - ], - ) - ) -) + +allowed_in_link_with_external_links = r"-?\w\$\.!\*'()/:" + +allowed_in_link = r"-?\w\$\.!\*'()/" links_list = {} in_use_links = {} @@ -81,7 +38,7 @@ def remove_extra_from_path(filepath): - """Given a path, remove unwanted path from a website link""" + """Given a path, remove unwanted path from a website link.""" return filepath.split(site_config.parent_web_directory)[1] @@ -114,7 +71,7 @@ def get_correct_link(path): "docx", "rtf", ]: - if re.search("(css|js)\?[\w\d]+", sort_of_extension): + if re.search(r"(css|js)\?\w+", sort_of_extension): # CSS & JavaScript: check for cache-disabling query string suffix, e.g style.min.css?f8be4c06 path = path.split("?")[0] # remove suffix else: @@ -135,7 +92,7 @@ def check_if_link_in_use(filepath, link): If not, verify that the link is not the same as the filepath and add it to the in use links map. """ - if not "previous" in link and not "versions" in link: + if "previous" not in link and "versions" not in link: if not in_use_links.get(link): new_file_name = remove_extra_from_path(filepath) @@ -146,19 +103,15 @@ def check_if_link_in_use(filepath, link): in_use_links[link] = True -def remove_subdirectory_from_web_directory(): - if site_config.subdirectory: - return site_config.web_directory.split(site_config.subdirectory)[0] - else: - return site_config.web_directory - - def internal_link_test(link): """Given a link, make sure that that it exists on the file system.""" + if site_config.subdirectory: + web_dir = site_config.web_directory.split(site_config.subdirectory)[0] + else: + web_dir = site_config.web_directory + # Get correct link path - path = get_correct_link(link) - - path = remove_subdirectory_from_web_directory() + path + path = web_dir + get_correct_link(link) # e.g: contacts.html -> contacts/index.html to_index_path = path @@ -201,9 +154,9 @@ def internal_external_link_checker(filepath, html_str): if ( "/versions/" in filepath ): # don't check links with data-test-ignore attribute, or live version link name, when on previous versions - linkregex = f'{prefix}\s?=\s?["\']([{allowed_in_link_with_external_links}]+)["\'](?! ?data-test-ignore="true")(?!>live version)' + linkregex = rf'{prefix}\s?=\s?["\']([{allowed_in_link_with_external_links}]+)["\'](?! ?data-test-ignore="true")(?!>live version)' else: - linkregex = f"{prefix}\s?=\s?[\"']([{allowed_in_link_with_external_links}]+)[\"']" + linkregex = rf"{prefix}\s?=\s?[\"']([{allowed_in_link_with_external_links}]+)[\"']" links = re.findall(linkregex, html_str) # check if link has a dest @@ -213,7 +166,7 @@ def internal_external_link_checker(filepath, html_str): # Add to relative links list if relative if is_relative: - if not link in relative_links: + if link not in relative_links: relative_links.append(link) # Get correct path @@ -262,7 +215,7 @@ def internal_link_checker(filepath, html_str): # find all links for prefix in ["href", "src"]: - links = re.findall(f"{prefix}\s?=\s?[\"']([{allowed_in_link}]+)[\"']", html_str) + links = re.findall(rf"{prefix}\s?=\s?[\"']([{allowed_in_link}]+)[\"']", html_str) # check if link has a dest for link in links: # Check if link is relative path @@ -270,7 +223,7 @@ def internal_link_checker(filepath, html_str): # Add to relative links list if relative if is_relative: - if not link in relative_links: + if link not in relative_links: relative_links.append(link) # Get correct path @@ -316,7 +269,7 @@ def check_unlinked_pages(filenames): """ unlinked_pages = [] for filename in filenames: - if not "previous" in filename and not "versions" in filename: + if "previous" not in filename and "versions" not in filename: # Check if it is deprecated if check_if_file_is_deprecated(filename): continue @@ -365,7 +318,7 @@ def check_links_on_page(filepath, check_external_links=False): with open(filepath, mode="r", encoding="utf8") as html: html_str = html.read() - if not "previous" in filepath and not "versions" in filepath: + if "previous" not in filepath and "versions" not in filepath: # Add redirects to in-use to avoid false positives if html_str.startswith(' {asset, relationship} for each asset targeted by the technique. @@ -262,6 +263,7 @@ def assets_targeted_by_techniques(srcs): """ return get_related(srcs, "attack-pattern", "targets", "x-mitre-asset") + # technique:malware def techniques_used_by_malware(srcs): """Return malware => {technique, relationship} for each technique used by the malware. diff --git a/modules/versions/versions.py b/modules/versions/versions.py index c7f965ba4e7..a2baea51ff4 100644 --- a/modules/versions/versions.py +++ b/modules/versions/versions.py @@ -14,8 +14,7 @@ def generate_versions(): - """Responsible for generating the versions pages""" - + """Responsible for generating the versions pages.""" # Move templates to templates directory util.buildhelpers.move_templates(versions_config.module_name, versions_config.versions_templates_path) @@ -53,30 +52,10 @@ def onerror(func, path, exc_info): # allowed characters inside of hyperlinks -allowed_in_link = "".join( - list( - map( - lambda s: s.strip(), - [ - " - ", - " ? ", - " \w ", - " \\ ", - " $ ", - " \. ", - " ! ", - " \* ", - " ' ", - " () ", - " / ", - ], - ) - ) -) - +allowed_in_link = r"-?\w\$\.!\*'()/" def versionPath(version): - # get the path of a given version + """Get the path of a given version.""" if "path" in version: return version["path"] else: @@ -84,7 +63,7 @@ def versionPath(version): def deploy(): - """Deploy previous versions to website directory""" + """Deploy previous versions to website directory.""" versions_config.prev_versions_deploy_folder = os.path.join( site_config.web_directory, versions_config.prev_versions_path ) @@ -176,7 +155,8 @@ def deploy_previous_version(version, repo): def archive(version_data, is_current=False): - """perform archival operations on a version in /prev_versions_path + """Perform archival operations on a version in /prev_versions_path. + - remove unnecessary files (.git, CNAME, preserved versions for that version) - replace links on all pages - add archived version banner to all pages @@ -236,10 +216,10 @@ def saferemove(path, type): html_str = html.read() # helper function to substitute links so that they point to /versions/ - dest_link_format = f"/{version_url_path}\g<1>" + dest_link_format = rf"/{version_url_path}\g<1>" def substitute(prefix, html_str): - fromstr = f"{prefix}=[\"'](?!\/versions\/)([{allowed_in_link}]+)[\"']" + fromstr = rf"{prefix}=[\"'](?!\/versions\/)([{allowed_in_link}]+)[\"']" tostr = f'{prefix}="{dest_link_format}"' return re.sub(fromstr, tostr, html_str) @@ -262,8 +242,8 @@ def substitute_redirection(prefix, html_str): # update versioning button to show the permalink site version, aka "back to main site" html_str = html_str.replace("version-button live", "version-button permalink") # update live version links on the versioning button - from_str = f"href=[\"']\/versions\/v[\w-]+\/([{allowed_in_link}]+)[\"'](.*)>[Ll]ive [Vv]ersion<\/a>" - to_str = f'href="/\g<1>"\g<2>>Live Version' + from_str = rf"href=[\"']\/versions\/v[\w-]+\/([{allowed_in_link}]+)[\"'](.*)>[Ll]ive [Vv]ersion<\/a>" + to_str = r'href="/\g<1>"\g<2>>Live Version' html_str = re.sub(from_str, to_str, html_str) # remove banner message if it is present @@ -323,7 +303,8 @@ def substitute_redirection(prefix, html_str): def build_alias(version, alias): - """build redirects from alias to version + """Build redirects from alias to version. + version is the path of the version, e.g "v5" alias is the alias to build, e.g "october2018" """ @@ -349,7 +330,7 @@ def build_alias(version, alias): def build_markdown(versions): - """build markdown for the versions list page""" + """Build markdown for the versions list page.""" # build urls versions["current"]["url"] = versionPath(versions["current"]) versions["current"]["changelog_label"] = " ".join(versions["current"]["changelog"].split("-")[1:]).title() diff --git a/modules/versions/versions_config.py b/modules/versions/versions_config.py index 0ca96ad20f1..8bc15a181f7 100644 --- a/modules/versions/versions_config.py +++ b/modules/versions/versions_config.py @@ -16,6 +16,4 @@ versions_repo = "https://github.com/mitre-attack/attack-website.git" versions_directory = "attack-versions" -versions_md = ( - "Title: Version History\n" "Template: versions/versions\n" "save_as: resources/versions/index.html\n" "data: " -) +versions_md = "Title: Version History\nTemplate: versions/versions\nsave_as: resources/versions/index.html\ndata: " diff --git a/modules/website_build/website_build_config.py b/modules/website_build/website_build_config.py index 46c9f50636e..b9c76d4a00b 100644 --- a/modules/website_build/website_build_config.py +++ b/modules/website_build/website_build_config.py @@ -57,7 +57,7 @@ } # ATT&CK overview -attack_index_md = "Title: ATT&CK Overview \n" "Template: general/attack-index \n" "save_as: index.html\n" "data: " +attack_index_md = "Title: ATT&CK Overview \nTemplate: general/attack-index \nsave_as: index.html\ndata: " # ATT&CK index markdown path attack_index_path = "content/pages/index.md" @@ -71,4 +71,4 @@ website_build_templates_path = "modules/website_build/templates/" # CHANGELOG md -changelog_md = "Title: Changelog\n" "Template: website_build/changelog\n" "save_as: resources/changelog.html\n\n" +changelog_md = "Title: Changelog\nTemplate: website_build/changelog\nsave_as: resources/changelog.html\n\n" diff --git a/pelicanconf.py b/pelicanconf.py index 4c74492c07b..320118d5e71 100644 --- a/pelicanconf.py +++ b/pelicanconf.py @@ -8,20 +8,20 @@ # read file with custom jinja filters import sys -sys.path.append('.') +sys.path.append(".") import custom_jinja_filters -AUTHOR = os.environ.get('PELICAN_AUTHOR', 'MITRE') -SITENAME = os.environ.get('PELICAN_SITENAME', 'ATT&CK') -SITEURL = os.environ.get('PELICAN_SITEURL', '') +AUTHOR = os.environ.get("PELICAN_AUTHOR", "MITRE") +SITENAME = os.environ.get("PELICAN_SITENAME", "ATT&CK") +SITEURL = os.environ.get("PELICAN_SITEURL", "") -PATH = 'content' +PATH = "content" -TIMEZONE = os.environ.get('PELICAN_TIMEZONE', 'America/New_York') +TIMEZONE = os.environ.get("PELICAN_TIMEZONE", "America/New_York") -DEFAULT_LANG = os.environ.get('PELICAN_DEFAULT_LANG','en') +DEFAULT_LANG = os.environ.get("PELICAN_DEFAULT_LANG", "en") -THEME = 'attack-theme' +THEME = "attack-theme" # Feed generation is usually not desired when developing FEED_ALL_ATOM = None @@ -31,18 +31,18 @@ AUTHOR_FEED_RSS = None DEFAULT_PAGINATION = False -STATIC_PATHS = ['docs'] -ARTICLE_PATHS = ['pages/updates'] +STATIC_PATHS = ["docs"] +ARTICLE_PATHS = ["pages/updates"] # Uncomment following line if you want document-relative URLs when developing RELATIVE_URLS = False JINJA_FILTERS = { - 'from_json': custom_jinja_filters.json.loads, - 'flatten_tree': custom_jinja_filters.flatten_tree, - 'clean_path': custom_jinja_filters.clean_path, - 'remove_whitespace': custom_jinja_filters.remove_whitespace, - 'escape_spaces': custom_jinja_filters.escape_spaces, - 'stixToHTML': custom_jinja_filters.stixToHTML, - 'permalink': custom_jinja_filters.permalink + "from_json": custom_jinja_filters.json.loads, + "flatten_tree": custom_jinja_filters.flatten_tree, + "clean_path": custom_jinja_filters.clean_path, + "remove_whitespace": custom_jinja_filters.remove_whitespace, + "escape_spaces": custom_jinja_filters.escape_spaces, + "stixToHTML": custom_jinja_filters.stixToHTML, + "permalink": custom_jinja_filters.permalink, } diff --git a/update-attack.py b/update-attack.py index 4b4fc048bdb..4e0565bdc32 100644 --- a/update-attack.py +++ b/update-attack.py @@ -230,7 +230,7 @@ def remove_from_menu(): ptr["run_module"]() end_time = time.time() util.buildhelpers.print_end(ptr["module_name"], start_time, end_time) - + # Print end of module update_end = time.time() util.buildhelpers.print_end("TOTAL Update Time", update_start, update_end)