Merge pull request #549 from mitre-attack/better-versioning

Better-versioning

Author: jondricek

.env.template

@@ -1,6 +1,7 @@
+ATTACK_VERSION_ARCHIVES="attack-version-archives"
+BANNER_ENABLED=True
+BANNER_MESSAGE="This is a custom instance of the MITRE ATT&CK Website. The official website can be found at <a href='https://attack.mitre.org'>attack.mitre.org</a>."
 STIX_LOCATION_ENTERPRISE="https://raw.githubusercontent.com/mitre/cti/master/enterprise-attack/enterprise-attack.json"
 STIX_LOCATION_MOBILE="https://raw.githubusercontent.com/mitre/cti/master/mobile-attack/mobile-attack.json"
 STIX_LOCATION_ICS="https://raw.githubusercontent.com/mitre/cti/master/ics-attack/ics-attack.json"
 STIX_LOCATION_PRE="https://raw.githubusercontent.com/mitre/cti/master/pre-attack/pre-attack.json"
-BANNER_ENABLED=True
-BANNER_MESSAGE="This is a custom instance of the MITRE ATT&CK Website. The official website can be found at <a href='https://attack.mitre.org'>attack.mitre.org</a>."

.github/workflows/static-code-analysis.yml

@@ -17,7 +17,7 @@ jobs:
           fetch-depth: 0  # Shallow clones should be disabled for a better relevancy of analysis
 
       - name: SonarCloud Scan
-        uses: SonarSource/sonarcloud-github-action@master
+        uses: SonarSource/sonarqube-scan-action@master
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}  # Needed to get PR information, if any
           SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}

.gitignore

@@ -39,3 +39,5 @@ node_modules/
 
 attack-style/dist/
 attack-style/docs/
+attack-version-archives/
+tmp/

archive-website.py

@@ -0,0 +1,58 @@
+import argparse
+import json
+import multiprocessing
+import os
+
+from loguru import logger
+
+from modules.versions.versions import create_version_archive
+
+
+def main():
+    """Process previous versions of ATT&CK website."""
+    parser = argparse.ArgumentParser(
+        description="Process previous versions of ATT&CK website and create version archives"
+    )
+    parser.add_argument(
+        "--archive-dir",
+        "-a",
+        default="attack-version-archives",
+        help="Directory where version archives will be created (default: attack-version-archives)",
+    )
+
+    args = parser.parse_args()
+    archive_dir = args.archive_dir
+
+    # Create archive directory if it doesn't exist
+    os.makedirs(archive_dir, exist_ok=True)
+
+    with open("data/versions.json", "r") as f:
+        version_json = json.load(f)
+
+    logger.info(f"Processing previous versions of ATT&CK website to {archive_dir}")
+
+    processes = []
+    for version_data in version_json["previous"]:
+        archive_path = os.path.join(archive_dir, f"website-{version_data['name']}.tar.gz")
+        if os.path.exists(archive_path):
+            logger.info(f"Archive already exists for {version_data['name']}: {archive_path} -- skipping.")
+            continue
+
+        p = multiprocessing.Process(
+            target=create_version_archive,
+            args=(version_data, archive_dir),
+            name=f"Process-{version_data['name']}",
+        )
+        processes.append(p)
+        p.start()
+        logger.info(f"Started process for {version_data['name']} (PID: {p.pid})")
+
+    for p in processes:
+        p.join()
+        logger.info(f"Process {p.name} for version finished (PID: {p.pid})")
+
+    logger.info("All versions processed")
+
+
+if __name__ == "__main__":
+    main()

data/versions.json

@@ -8,7 +8,6 @@
     "previous": [
         {
             "name": "v16.1",
-            "aliases": [],
             "date_start": "October 31, 2024",
             "date_end": "April 21, 2025",
             "changelog": "updates-october-2024",
@@ -17,7 +16,6 @@
         },
         {
             "name": "v15.1",
-            "aliases": [],
             "date_start": "April 23, 2024",
             "date_end": "October 30, 2024",
             "changelog": "updates-april-2024",
@@ -26,7 +24,6 @@
         },
         {
             "name": "v14.1",
-            "aliases": [],
             "date_start": "October 31, 2023",
             "date_end": "April 22, 2024",
             "changelog": "updates-october-2023",
@@ -35,7 +32,6 @@
         },
         {
             "name": "v13.1",
-            "aliases": [],
             "date_start": "April 25, 2023",
             "date_end": "October 30, 2023",
             "changelog": "updates-april-2023",
@@ -44,7 +40,6 @@
         },
         {
             "name": "v12.1",
-            "aliases": [],
             "date_start": "October 25, 2022",
             "date_end": "April 24, 2023",
             "changelog": "updates-october-2022",
@@ -53,7 +48,6 @@
         },
         {
             "name": "v11.3",
-            "aliases": [],
             "date_start": "April 25, 2022",
             "date_end": "October 24, 2022",
             "changelog": "updates-april-2022",
@@ -62,7 +56,6 @@
         },
         {
             "name": "v10.1",
-            "aliases": [],
             "date_start": "October 21, 2021",
             "date_end": "April 24, 2022",
             "changelog": "updates-october-2021",
@@ -71,7 +64,6 @@
         },
         {
             "name": "v9.0",
-            "aliases": [],
             "date_start": "April 29, 2021",
             "date_end": "October 20, 2021",
             "changelog": "updates-april-2021",
@@ -80,36 +72,22 @@
         },
         {
             "name": "v8.2",
-            "aliases": [],
             "date_start": "October 27, 2020",
             "date_end": "April 28, 2021",
             "changelog": "updates-october-2020",
             "cti_url": "https://github.com/mitre/cti/releases/tag/ATT%26CK-v8.2",
             "commit": "6073984f48327644bd97256aa62ecf142b690200"
-
         },
         {
             "name": "v7.2",
-            "aliases": [],
             "date_start": "July 8, 2020",
             "date_end": "October 26, 2020",
             "changelog": "updates-july-2020",
             "cti_url": "https://github.com/mitre/cti/releases/tag/ATT%26CK-v7.2",
             "commit": "11a8bce02433810457b90f61e66c5e4a609cbd66"
         },
-        {
-            "name": "v7.0-beta",
-            "path": "v7-beta",
-            "aliases": [],
-            "date_start": "March 31, 2020",
-            "date_end": "July 7, 2020",
-            "changelog": "updates-march-2020",
-            "cti_url": "https://github.com/mitre/cti/releases/tag/ATT%26CK-v7.0-beta",
-            "commit": "d086905c3c2242b9f76c893e4ad79ba39ec9ae26"
-        },
         {
             "name": "v6.3",
-            "aliases": [],
             "date_start": "October 24, 2019",
             "date_end": "March 30, 2020",
             "changelog": "updates-october-2019",
@@ -118,7 +96,6 @@
         },
         {
             "name": "v5.2",
-            "aliases": ["july2019"],
             "date_start": "July 31, 2019",
             "date_end": "October 23, 2019",
             "changelog": "updates-july-2019",
@@ -127,7 +104,6 @@
         },
         {
             "name": "v4.0",
-            "aliases": ["april2019"],
             "date_start": "April 30, 2019",
             "date_end": "July 30, 2019",
             "changelog": "updates-april-2019",
@@ -136,7 +112,6 @@
         },
         {
             "name": "v3.0",
-            "aliases": ["october2018"],
             "date_start": "October 23, 2018",
             "date_end": "April 29, 2019",
             "changelog": "updates-october-2018",

modules/__init__.py

@@ -14,9 +14,6 @@ def sort_menu_by_priority():
 def sort_run_ptr_by_priority():
     global run_ptr
     run_ptr = sorted(run_ptr, key=lambda k: k["priority"])
-    print(
-        f"Building website using the following modules in this order: {[pointer_info['module_name'] for pointer_info in run_ptr]}"
-    )
 
 
 def check_redirections(redirections_list):

modules/assets/assets.py

@@ -2,12 +2,10 @@
 import json
 import os
 
-from loguru import logger
-
 from modules import util
 
-from . import assets_config
 from .. import site_config
+from . import assets_config
 
 
 def generate_assets():
@@ -22,11 +20,6 @@ def generate_assets():
     if not os.path.isdir(assets_config.asset_markdown_path):
         os.mkdir(assets_config.asset_markdown_path)
 
-    # Generate redirections
-    util.buildhelpers.generate_redirections(
-        redirections_filename=assets_config.assets_redirection_location, redirect_md=site_config.redirect_md
-    )
-
     # Generates the markdown files to be used for page generation
     assets_generated = generate_markdown_files()
 
@@ -35,9 +28,7 @@ def generate_assets():
 
 
 def generate_markdown_files():
-    """
-    Responsible for generating asset index page and getting shared data for all assets.
-    """
+    """Responsible for generating asset index page and getting shared data for all assets."""
     has_asset = False
 
     asset_list = util.relationshipgetters.get_asset_list()
@@ -70,7 +61,6 @@ def generate_markdown_files():
 
 def generate_asset_md(asset, side_menu_data, notes):
     """Responsible for generating markdown of all assets."""
-
     attack_id = util.buildhelpers.get_attack_id(asset)
 
     if not attack_id:

modules/assets/assets_redirections.json

@@ -1,3 +1,5 @@
+from loguru import logger
+
 from . import blog_config
 
 
@@ -14,3 +16,6 @@ def get_menu():
         "priority": blog_config.priority,
         "children": [],
     }
+
+def run_module():
+    logger.info("Running Blog module...")

modules/blog/__init__.py

@@ -6,15 +6,12 @@
 
 from modules import util
 
-from . import campaigns_config
 from .. import site_config
+from . import campaigns_config
 
 
 def generate_campaigns():
-    """
-    Responsible for verifying campaign directory and starting off campaign markdown generation.
-    """
-
+    """Responsible for verifying campaign directory and starting off campaign markdown generation."""
     # Create content pages directory if does not already exist
     util.buildhelpers.create_content_pages_dir()
 
@@ -25,12 +22,6 @@ def generate_campaigns():
     if not os.path.isdir(campaigns_config.campaign_markdown_path):
         os.mkdir(campaigns_config.campaign_markdown_path)
 
-    # TODO resolve infinite redirect loop when run locally. Needs further testing before code removal.
-    # Generate redirections
-    util.buildhelpers.generate_redirections(
-        redirections_filename=campaigns_config.campaigns_redirection_location, redirect_md=site_config.redirect_md
-    )
-
     # Generates the markdown files to be used for page generation
     campaigns_generated = generate_markdown_files()
 
@@ -39,10 +30,7 @@ def generate_campaigns():
 
 
 def generate_markdown_files():
-    """
-    Responsible for generating campaign index page and getting shared data for all campaigns.
-    """
-
+    """Responsible for generating campaign index page and getting shared data for all campaigns."""
     has_campaign = False
 
     campaign_list = util.relationshipgetters.get_campaign_list()
@@ -79,7 +67,6 @@ def generate_markdown_files():
 
 def generate_campaign_md(campaign, side_menu_data, notes):
     """Responsible for generating markdown of all campaigns."""
-
     attack_id = util.buildhelpers.get_attack_id(campaign)
 
     if attack_id: