adding dscription on technologies collection

Author: maceto

scripts/script_create_descriptions_json.py

@@ -0,0 +1,38 @@
+import os
+import json
+
+##################################################################################################
+#
+# This script read all techonolgoies descriptions from wappalyzer/src/technologies directory
+# and creates a JSON file with the keys and descriptions
+#
+##################################################################################################
+
+# Define the directory path
+directory_path = "../../wappalyzer/src/technologies"
+
+# Initialize a dictionary to store the keys and descriptions
+key_description_dict = {}
+
+# Loop through all the files in the directory
+for filename in os.listdir(directory_path):
+    if filename.endswith(".json"):
+        file_path = os.path.join(directory_path, filename)
+        
+        # Open and parse the JSON file
+        with open(file_path, 'r', encoding='utf-8') as json_file:
+            data = json.load(json_file)
+            
+            # Loop through the keys in the JSON data
+            for key, value in data.items():
+                if 'description' in value:
+                    key_description_dict[key] = value['description']
+
+# Define the path for the output JSON file
+output_file_path = "key_description.json"
+
+# Save the keys and descriptions in the output JSON file
+with open(output_file_path, 'w', encoding='utf-8') as output_file:
+    json.dump(key_description_dict, output_file, indent=4)
+
+print(f"Keys and descriptions saved to {output_file_path}")

scripts/script_save_tech_descriptions.py

@@ -0,0 +1,29 @@
+import os
+import json
+from google.cloud import firestore
+
+# Initialize Firestore client
+# Replace 'your-project-id' with your actual Firestore project ID
+# You must also set up authentication for your project.
+db = firestore.Client()
+
+# Define the path to the JSON file
+json_file_path = "key_description.json"
+
+# Read the JSON data from the file
+with open(json_file_path, 'r', encoding='utf-8') as json_file:
+    data = json.load(json_file)
+
+# Define the Firestore collection reference
+collection_ref = db.collection(u'technologies-list')
+
+# Loop through the data and save it to Firestore
+for key, description in data.items():
+    document_data = {
+        'name': key,       
+        'description': description
+    }
+
+    collection_ref.document().set(document_data)
+
+print("Data has been saved to Firestore.")

scripts/script_technologies.py

@@ -1,3 +1,12 @@
+################################################################################
+# Description: This script queries the BigQuery table "technologies" 
+# and inserts the result into 
+# the Firestore collection "technologies".
+#
+# Parameters: start_date (optional), end_date (optional)
+# Usage example: python scripts/script_technologies.py 2020-01-01 2020-12-31
+################################################################################
+
 import sys
 from google.cloud import bigquery
 from google.cloud import firestore
@@ -16,16 +25,6 @@ def execute_query_and_insert_result(start_date, end_date):
     # Set up Firestore client
     firestore_client = firestore.Client()
 
-    # Define the BigQuery query with optional parameters
-    # query = """
-    #     SELECT
-    #       *
-    #     FROM
-    #       `httparchive.core_web_vitals.technologies`
-    #     WHERE
-    #       1=1
-    # """
-
     query = """
         SELECT
             date,
@@ -44,7 +43,6 @@ def execute_query_and_insert_result(start_date, end_date):
             geo = 'ALL' AND
             rank = 'ALL'
     """
-
     # Construct the WHERE clause based on the provided parameters
     if start_date and end_date:
         query += f" AND date >= '{start_date}' AND date <= '{end_date}'"
@@ -56,12 +54,26 @@ def execute_query_and_insert_result(start_date, end_date):
     results = query_job.result()
 
     # Create a new Firestore document for each result and insert it into the "technologies" collection
-    collection_ref = firestore_client.collection('technologies')
-    print(results)
+    collection_ref = firestore_client.collection(u'technologies')
+    
+    tech_collection_ref = firestore_client.collection(u'technologies-list')
+
     for row in results:
 
         item = dict(row.items())
+
+        # Query the techonologies-list collection for the description
+        #
+        tech_query = tech_collection_ref.where('name', '==', row['technology'])
+        tech_query = tech_query.limit(1)
+        tech_results = tech_query.stream()
+        technology = {}
+        for tech in tech_results:
+            technology = tech.to_dict()
+
+        # overriding BQ fields
         item['date'] = str(row['date'])
+        item['description'] = technology.get('description','')
 
         print(item)