feat(search): KTL-1516: use TeamCity external deps for stats

Author: zoobestik

.teamcity/builds/kotlinlang/buidTypes/BuildSearchIndex.kt

@@ -10,7 +10,13 @@ import jetbrains.buildServer.configs.kotlin.buildSteps.ScriptBuildStep
 import jetbrains.buildServer.configs.kotlin.buildSteps.script
 import jetbrains.buildServer.configs.kotlin.triggers.schedule
 import vcsRoots.KotlinLangOrg
+import java.io.File
+import java.nio.file.Paths
 
+private fun readScript(name: String): String {
+  val file = File(Paths.get("scripts/$name.mjs").toAbsolutePath().toString())
+  return file.readText()
+}
 
 object BuildSearchIndex : BuildType({
   name = "Build Site Search Index"
@@ -24,6 +30,11 @@ object BuildSearchIndex : BuildType({
     param("env.WH_SEARCH_KEY", "%ALGOLIA_WRITE_API_KEY%")
   }
 
+  artifactRules = """
+    page_views_list.json
+    page_views_map.json
+  """.trimIndent()
+
   vcs {
     root(KotlinLangOrg)
 
@@ -33,6 +44,19 @@ object BuildSearchIndex : BuildType({
 
   steps {
     script {
+      name = "Prepare page views"
+      scriptContent = """
+        #!/usr/bin/env bash
+        ":" //# comment; exec /usr/bin/env node --input-type=module - "${'$'}@" < "${'$'}0"
+        
+        ${readScript("stats/pageviews")}
+      """.trimIndent()
+      dockerImage = "node:lts-slim"
+      dockerImagePlatform = ScriptBuildStep.ImagePlatform.Linux
+      dockerPull = true
+    }
+    script {
+        name = "Push search index"
       scriptContent = """
         #!/bin/bash
         
@@ -71,6 +95,14 @@ object BuildSearchIndex : BuildType({
       onDependencyFailure = FailureAction.FAIL_TO_START
       onDependencyCancel = FailureAction.CANCEL
     }
+
+    artifacts(AbsoluteId("WebTeam_BuildsForDeploymentJetBrainsCom_Algolia_PageViewsFromGoogle")) {
+      buildRule = lastSuccessful()
+      artifactRules = """
+        +:unique_pageviews_pages_000000000000.json => data
+      """.trimIndent()
+    }
+
     dependency(BuildSitePages) {
       snapshot {}
 

.teamcity/scripts/stats/pageviews.mjs

@@ -0,0 +1,72 @@
+import { open } from 'node:fs/promises';
+
+const INPUT_FILE_PATH = 'data/unique_pageviews_pages_000000000000.json';
+const input = await open(INPUT_FILE_PATH, 'r');
+
+async function openReportFile() {
+    const file = await open('page_views_list.json', 'w');
+    await file.truncate(0);
+    return file;
+}
+
+const [listViews, mapViews] = await Promise.all([
+    openReportFile('page_views_list.json'),
+    openReportFile('page_views_map.json'),
+]);
+
+async function append(line) {
+    const { webpage: url, unique_pageviews: views } = JSON.parse(line);
+
+    const pageviews = Number(views);
+
+    if (views === '' || isNaN(pageviews)) {
+        console.warn(`${url} has incorrect unique_pageviews=${views}`);
+        return;
+    }
+
+    if (pageviews < 1) return;
+    if (!(new URL(url).host.includes('kotlinlang.org'))) return;
+
+    await Promise.all([
+        listViews.appendFile(JSON.stringify({ url, pageviews }) + ','),
+        mapViews.appendFile(`${JSON.stringify(url)}: ${pageviews},`)
+    ]);
+}
+
+const lines = [];
+
+await Promise.all([
+    listViews.write('['),
+    mapViews.write('{')
+]);
+
+const readlineInterface = input.readLines();
+
+readlineInterface.on('line', line => {
+    lines.push(append(line));
+});
+
+const waitInputRead = new Promise(resolve => {
+    readlineInterface.on('close', () => {
+        resolve();
+    });
+});
+
+await waitInputRead;
+await Promise.all(lines);
+
+async function replaceLastCharacter(file, ch) {
+    const { size } = await file.stat();
+    file.write(ch, size - 1);
+}
+
+await Promise.all([
+    replaceLastCharacter(listViews, ']'),
+    replaceLastCharacter(mapViews, '}'),
+]);
+
+await Promise.all([
+    input.close(),
+    listViews.close(),
+    mapViews.close(),
+]);