feat(search): KTL-1516: use separate build for collecting page_views

Author: zoobestik

.teamcity/builds/kotlinlang/buidTypes/BuildSearchIndex.kt

@@ -10,31 +10,18 @@ import jetbrains.buildServer.configs.kotlin.buildSteps.ScriptBuildStep
 import jetbrains.buildServer.configs.kotlin.buildSteps.script
 import jetbrains.buildServer.configs.kotlin.triggers.schedule
 import vcsRoots.KotlinLangOrg
-import java.io.File
-import java.nio.file.Paths
-
-private fun readScript(name: String): String {
-  val file = File(Paths.get("scripts/$name.mjs").toAbsolutePath().toString())
-  return file.readText()
-}
 
 object BuildSearchIndex : BuildType({
   name = "Build Site Search Index"
   description = "Build search index for Algolia using Google Analytics data"
 
   params {
-    param("env.KEY_FILE_LOCATION", "/secrets/google-credentials.json")
     param("virtualenv.folder", "_environment")
     param("env.WH_INDEX_NAME", SEARCH_INDEX_NAME)
     param("env.WH_SEARCH_USER", SEARCH_APP_ID)
     param("env.WH_SEARCH_KEY", "%ALGOLIA_WRITE_API_KEY%")
   }
 
-  artifactRules = """
-    page_views_list.json
-    page_views_map.json
-  """.trimIndent()
-
   vcs {
     root(KotlinLangOrg)
 
@@ -43,26 +30,10 @@ object BuildSearchIndex : BuildType({
   }
 
   steps {
-    script {
-      name = "Prepare page views"
-      scriptContent = """
-        #!/usr/bin/env bash
-        ":" //# comment; exec /usr/bin/env node --input-type=module - "${'$'}@" < "${'$'}0"
-        
-        ${readScript("stats/pageviews")}
-      """.trimIndent()
-      dockerImage = "node:lts-slim"
-      dockerImagePlatform = ScriptBuildStep.ImagePlatform.Linux
-      dockerPull = true
-    }
     script {
         name = "Push search index"
       scriptContent = """
-        #!/bin/bash
-        
-        ## refresh packages
-        pip install -r requirements.txt
-        
+        #!/bin/bash 
         python kotlin-website.py index
       """.trimIndent()
       dockerImage = "%dep.Kotlin_KotlinSites_Builds_KotlinlangOrg_BuildPythonContainer.kotlin-website-image%"
@@ -96,11 +67,14 @@ object BuildSearchIndex : BuildType({
       onDependencyCancel = FailureAction.CANCEL
     }
 
-    artifacts(AbsoluteId("WebTeam_BuildsForDeploymentJetBrainsCom_Algolia_PageViewsFromGoogle")) {
-      buildRule = lastSuccessful()
-      artifactRules = """
-        +:unique_pageviews_pages_000000000000.json => data
-      """.trimIndent()
+    dependency(PageViews) {
+      snapshot {}
+
+      artifacts {
+        artifactRules = """
+          page_views_map.json => data/
+        """.trimIndent()
+      }
     }
 
     dependency(BuildSitePages) {

.teamcity/builds/kotlinlang/buidTypes/PageViews.kt

@@ -1,8 +1,58 @@
 package builds.kotlinlang.buidTypes
 
+import jetbrains.buildServer.configs.kotlin.AbsoluteId
 import jetbrains.buildServer.configs.kotlin.BuildType
+import jetbrains.buildServer.configs.kotlin.buildSteps.ScriptBuildStep
+import jetbrains.buildServer.configs.kotlin.buildSteps.script
+import jetbrains.buildServer.configs.kotlin.triggers.finishBuildTrigger
+import java.io.File
+import java.nio.file.Paths
+
+private fun readScript(name: String): String {
+    val file = File(Paths.get("scripts/$name.mjs").toAbsolutePath().toString())
+    return file.readText()
+}
+
+private val pageViewsCollectId = AbsoluteId("WebTeam_BuildsForDeploymentJetBrainsCom_Algolia_PageViewsFromGoogle")
 
 object PageViews : BuildType({
     name = "Fetch Page Views"
     description = "Build data files with page views statistics for kotlin websites"
+
+    artifactRules = """
+        page_views_list.json
+        page_views_map.json
+      """.trimIndent()
+
+    triggers {
+        finishBuildTrigger {
+            buildType = pageViewsCollectId.absoluteId
+            branchFilter = "+:<default>"
+            successfulOnly = true
+        }
+    }
+
+    steps {
+        script {
+            name = "Prepare page views"
+            scriptContent = """
+                #!/usr/bin/env bash
+                ":" //# comment; exec /usr/bin/env node --input-type=module - "${'$'}@" < "${'$'}0"
+                
+                ${readScript("stats/pageviews")}
+            """.trimIndent()
+            dockerImage = "node:lts-slim"
+            dockerImagePlatform = ScriptBuildStep.ImagePlatform.Linux
+            dockerPull = true
+        }
+    }
+
+    dependencies {
+        artifacts(pageViewsCollectId) {
+            buildRule = lastSuccessful()
+            artifactRules = """
+                +:unique_pageviews_pages_000000000000.json => data
+            """.trimIndent()
+        }
+    }
 })

.teamcity/scripts/stats/pageviews.mjs

@@ -3,8 +3,8 @@ import { open } from 'node:fs/promises';
 const INPUT_FILE_PATH = 'data/unique_pageviews_pages_000000000000.json';
 const input = await open(INPUT_FILE_PATH, 'r');
 
-async function openReportFile() {
-    const file = await open('page_views_list.json', 'w');
+async function openReportFile(name) {
+    const file = await open(name, 'w');
     await file.truncate(0);
     return file;
 }
@@ -14,59 +14,61 @@ const [listViews, mapViews] = await Promise.all([
     openReportFile('page_views_map.json'),
 ]);
 
-async function append(line) {
-    const { webpage: url, unique_pageviews: views } = JSON.parse(line);
+try {
+    async function append(line) {
+        const { webpage: url, unique_pageviews: views } = JSON.parse(line);
 
-    const pageviews = Number(views);
+        const pageviews = Number(views);
 
-    if (views === '' || isNaN(pageviews)) {
-        console.warn(`${url} has incorrect unique_pageviews=${views}`);
-        return;
+        if (views === '' || isNaN(pageviews)) {
+            console.warn(`${url} has incorrect unique_pageviews=${views}`);
+            return;
+        }
+
+        if (pageviews < 1) return;
+        if (!(new URL(url).host.includes('kotlinlang.org'))) return;
+
+        await Promise.all([
+            listViews.appendFile(JSON.stringify({ url, pageviews }) + ','),
+            mapViews.appendFile(`${JSON.stringify(url)}: ${pageviews},`),
+        ]);
     }
 
-    if (pageviews < 1) return;
-    if (!(new URL(url).host.includes('kotlinlang.org'))) return;
+    const lines = [];
 
     await Promise.all([
-        listViews.appendFile(JSON.stringify({ url, pageviews }) + ','),
-        mapViews.appendFile(`${JSON.stringify(url)}: ${pageviews},`)
+        listViews.write('['),
+        mapViews.write('{'),
     ]);
-}
 
-const lines = [];
-
-await Promise.all([
-    listViews.write('['),
-    mapViews.write('{')
-]);
+    const readlineInterface = input.readLines();
 
-const readlineInterface = input.readLines();
-
-readlineInterface.on('line', line => {
-    lines.push(append(line));
-});
-
-const waitInputRead = new Promise(resolve => {
-    readlineInterface.on('close', () => {
-        resolve();
+    readlineInterface.on('line', line => {
+        lines.push(append(line));
     });
-});
 
-await waitInputRead;
-await Promise.all(lines);
+    const waitInputRead = new Promise(resolve => {
+        readlineInterface.on('close', () => {
+            resolve();
+        });
+    });
 
-async function replaceLastCharacter(file, ch) {
-    const { size } = await file.stat();
-    file.write(ch, size - 1);
-}
+    await waitInputRead;
+    await Promise.all(lines);
 
-await Promise.all([
-    replaceLastCharacter(listViews, ']'),
-    replaceLastCharacter(mapViews, '}'),
-]);
+    async function replaceLastCharacter(file, ch) {
+        const { size } = await file.stat();
+        file.write(ch, size - 1);
+    }
 
-await Promise.all([
-    input.close(),
-    listViews.close(),
-    mapViews.close(),
-]);
+    await Promise.all([
+        replaceLastCharacter(listViews, ']'),
+        replaceLastCharacter(mapViews, '}'),
+    ]);
+} finally {
+    await Promise.all([
+        input.close(),
+        listViews.close(),
+        mapViews.close(),
+    ]);
+}

src/search.py

@@ -13,7 +13,7 @@
 def get_page_views_statistic() -> Dict[str, int]:
     print("Acquiring page view statistic")
 
-    file = open("page_views_map.json", "r")
+    file = open("data/page_views_map.json", "r")
     page_views = json.load(file)
     file.close()