Tech Report: Technologies total origins from crawl (#47)

* full adoption

* sql fix

* Update definitions/output/reports/cwv_tech_technologies.js

Co-authored-by: Barry Pollard <[email protected]>

* Update definitions/output/reports/cwv_tech_technologies.js

Co-authored-by: Barry Pollard <[email protected]>

* Update definitions/output/reports/cwv_tech_technologies.js

Co-authored-by: Barry Pollard <[email protected]>

* review

* cleanup

* sorted arrays

---------

Co-authored-by: Barry Pollard <[email protected]>

Author: max-ostapenko

definitions/output/reports/cwv_tech_technologies.js

@@ -6,19 +6,73 @@ publish('cwv_tech_technologies', {
   tags: ['crux_ready']
 }).query(ctx => `
 /* {"dataform_trigger": "report_cwv_tech_complete", "name": "technologies", "type": "dict"} */
+WITH pages AS (
+  SELECT DISTINCT
+    client,
+    root_page,
+    tech.technology
+  FROM ${ctx.ref('crawl', 'pages')},
+    UNNEST(technologies) AS tech
+  WHERE
+    date = '${pastMonth}'
+    ${constants.devRankFilter}
+),
+
+tech_origins AS (
+  SELECT
+    client,
+    technology,
+    COUNT(DISTINCT root_page) AS origins
+  FROM pages
+  GROUP BY
+    client,
+    technology
+),
+
+technologies AS (
+  SELECT
+    name AS technology,
+    description,
+    STRING_AGG(DISTINCT category, ', ' ORDER BY category ASC) AS category,
+    categories AS category_obj,
+    NULL AS similar_technologies
+  FROM ${ctx.ref('wappalyzer', 'technologies')},
+    UNNEST(categories) AS category
+  GROUP BY
+    technology,
+    description,
+    categories
+),
+
+total_pages AS (
+  SELECT
+    client,
+    COUNT(DISTINCT root_page) AS origins
+  FROM pages
+  GROUP BY client
+)
+
 SELECT
   client,
-  app AS technology,
+  technology,
   description,
   category,
-  SPLIT(category, ",") AS category_obj,
+  category_obj,
+  similar_technologies,
+  origins
+FROM tech_origins
+INNER JOIN technologies
+USING(technology)
+
+UNION ALL
+
+SELECT
+  client,
+  'ALL' AS technology,
+  NULL AS description,
+  NULL AS category,
+  NULL AS category_obj,
   NULL AS similar_technologies,
   origins
-FROM ${ctx.ref('core_web_vitals', 'technologies')}
-LEFT JOIN ${ctx.ref('wappalyzer', 'technologies')}
-ON app = name
-WHERE date = '${pastMonth}' AND
-  geo = 'ALL' AND
-  rank = 'ALL'
-ORDER BY origins DESC
+FROM total_pages
 `)