Docstat: CLI tool to fetch metrics from the Kusto API (#55917)

Author: sarahs

package-lock.json

@@ -256,6 +256,7 @@
     "ajv": "^8.17.1",
     "ajv-errors": "^3.0.0",
     "ajv-formats": "^3.0.1",
+    "azure-kusto-data": "^7.0.0",
     "bottleneck": "2.19.5",
     "boxen": "8.0.1",
     "cheerio": "^1.0.0-rc.12",

package.json

@@ -0,0 +1,19 @@
+# Kusto tooling
+
+CLI tools to fetch data from the Kusto API.
+
+## Installation and authentication
+
+1. Install the Azure CLI with `brew install azure-cli`.
+    * If you have the option to **not** update all your brew packages, choose that, or it will take a really long time.
+1. Run `az login`.
+    * You'll have to run `az login` whenever your session expires. The sessions are fairly long lasting.
+1. Enter your `<username>@githubazure.com` credentials.
+    * These will get cached for future logins.
+1. At the prompt in Terminal asking which subscription you want to use, just press Enter to choose the default.
+1. Open or create an `.env` file in the root directory of your checkout (this file is already in `.gitignore`).
+1. Add the `KUSTO_CLUSTER` and `KUSTO_DATABASE` values to the `.env`.
+    ```
+    KUSTO_CLUSTER='<value>'
+    KUSTO_DATABASE='<value>'
+    ```

src/metrics/README.md

@@ -0,0 +1,28 @@
+const dateOpts = {
+  year: 'numeric',
+  month: 'long',
+  day: 'numeric',
+}
+
+// Default to 30 days ago if a range option is not provided
+export function getDates(range = '30') {
+  // Get current datetime in ISO format
+  const today = new Date()
+  const todayISO = today.toISOString()
+
+  // Get datetime from N days ago in ISO format
+  const daysAgo = getDaysAgo(Number(range))
+  const daysAgoISO = daysAgo.toISOString()
+
+  return {
+    endDate: todayISO,
+    startDate: daysAgoISO,
+    friendlyRange: `${daysAgo.toLocaleDateString('en-US', dateOpts)} - ${today.toLocaleDateString('en-US', dateOpts)}`,
+  }
+}
+
+function getDaysAgo(range) {
+  const daysAgo = new Date()
+  daysAgo.setDate(daysAgo.getDate() - range)
+  return daysAgo
+}

src/metrics/lib/dates.js

@@ -0,0 +1,40 @@
+import { Client as KustoClient, KustoConnectionStringBuilder } from 'azure-kusto-data'
+
+import dotenv from 'dotenv'
+dotenv.config()
+if (!(process.env.KUSTO_CLUSTER || process.env.KUSTO_DATABASE)) {
+  console.error(`Add KUSTO_CLUSTER and KUSTO_DATABASE to your .env file`)
+  process.exit(0)
+}
+const KUSTO_CLUSTER = process.env.KUSTO_CLUSTER
+const KUSTO_DATABASE = process.env.KUSTO_DATABASE
+
+export function getKustoClient() {
+  let client
+  try {
+    const kcsb = KustoConnectionStringBuilder.withAzLoginIdentity(KUSTO_CLUSTER)
+    client = new KustoClient(kcsb)
+  } catch (error) {
+    console.error('Error connecting to Kusto')
+    console.error(error)
+  }
+  return client
+}
+
+export async function runQuery(pathToFetch, query, client, queryType, verbose = false) {
+  // Display query if verbose mode is on
+  if (verbose) {
+    console.log(`\n--- EXECUTING QUERY FOR "${queryType.toUpperCase()}" ---`)
+    console.log(query)
+    console.log('----------------------\n')
+  }
+
+  const results = await client.execute(KUSTO_DATABASE, query)
+
+  if (results.primaryResults.length === 0) {
+    console.log(`No data found for URL: ${pathToFetch}`)
+    return null
+  }
+
+  return results
+}

src/metrics/lib/kusto-client.js

@@ -0,0 +1,39 @@
+import { runQuery } from '#src/metrics/lib/kusto-client.js'
+import { SHARED_DECLARATIONS, SHARED_FILTERS } from '#src/metrics/queries/constants.js'
+
+const QUERY_TYPE = 'bounces'
+
+export async function getBounces(
+  pathToFetch,
+  client,
+  dates,
+  version = null,
+  verbose = false,
+  queryType = QUERY_TYPE,
+) {
+  const query = getBouncesQuery(pathToFetch, dates, version)
+  const results = await runQuery(pathToFetch, query, client, queryType, verbose)
+  const data = JSON.parse(results.primaryResults[0].toString()).data[0]
+  // Extract Bounces
+  const bounces = data.Bounces
+  return bounces
+}
+
+export function getBouncesQuery(pathToFetch, dates, version) {
+  return `
+    ${SHARED_DECLARATIONS(pathToFetch, dates, version)}
+    let _exits = () {
+        docs_v0_exit_event
+    ${SHARED_FILTERS}
+    };
+    _exits
+    | summarize Bounces=round(
+        countif(exit_scroll_length < 0.1 and exit_visit_duration < 5) / 
+          toreal(count()), 
+        2
+      )
+    | project Bounces=strcat(toint(
+        Bounces * 100
+      ), '%')
+  `
+}

src/metrics/queries/bounces.js

@@ -0,0 +1,24 @@
+// SHARED QUERY CONSTANTS
+export const SHARED_DECLARATIONS = (path, dates, version) =>
+  `
+    let _article = dynamic(['${Array.isArray(path) ? path.join("', '") : path}']);
+    let _articleType = dynamic(null);
+    let _endTime = datetime(${dates.endDate || null});
+    let _language = dynamic(null);
+    let _pageType = dynamic(null);
+    let _product = dynamic(null);
+    let _startTime = datetime(${dates.startDate || null});
+    let _version = dynamic(${version ? `['${version}']` : null});
+`.trim()
+
+export const SHARED_FILTERS = `
+        | where timestamp between (_startTime .. _endTime)
+        | where context.hostname == 'docs.github.com'
+        | where abs(totimespan(context.created - timestamp)) < 1h
+        | where isempty(_language) or tostring(context.path_language) in (_language)
+        | where isempty(_version) or tostring(context.path_version) in (_version)
+        | where isempty(_article) or context.path_article has_any (_article)
+        | where isempty(_product) or tostring(context.path_product) in (_product)
+        | where isempty(_pageType) or tostring(context.page_document_type) in (_pageType)
+        | where isempty(_articleType) or tostring(context.page_type) in (_articleType)
+`.trim()

src/metrics/queries/constants.js

@@ -0,0 +1,38 @@
+import { runQuery } from '#src/metrics/lib/kusto-client.js'
+import { SHARED_DECLARATIONS, SHARED_FILTERS } from '#src/metrics/queries/constants.js'
+
+const QUERY_TYPE = 'exits'
+
+export async function getExitsToSupport(
+  pathToFetch,
+  client,
+  dates,
+  version = null,
+  verbose = false,
+  queryType = QUERY_TYPE,
+) {
+  const query = getExitsQueryStatement(pathToFetch, dates, version)
+  const results = await runQuery(pathToFetch, query, client, queryType, verbose)
+  const data = JSON.parse(results.primaryResults[0].toString()).data[0]
+  // Extract Column1
+  const exitsToSupport = data.Column1
+  return exitsToSupport
+}
+
+export function getExitsQueryStatement(pathToFetch, dates, version) {
+  return `
+    ${SHARED_DECLARATIONS(pathToFetch, dates, version)}
+    let _links = () {
+        docs_v0_link_event
+    ${SHARED_FILTERS}
+    };
+    _links
+    | where isempty(link_samesite) or link_samesite == false
+    | where link_samepage != true // allow false or null
+    | extend link_url_parsed=parse_url(link_url)
+    | extend IsSupport=tostring(link_url_parsed.Host) == "support.github.com" 
+      and tostring(link_url_parsed.path) != "/enterprise/server-upgrade"
+    | summarize Ratio=round(countif(IsSupport) / toreal(count()), 2)
+    | project strcat(toint(Ratio * 100), '%')
+  `
+}

src/metrics/queries/exits-to-support.js

@@ -0,0 +1,42 @@
+import { runQuery } from '#src/metrics/lib/kusto-client.js'
+import { SHARED_DECLARATIONS, SHARED_FILTERS } from '#src/metrics/queries/constants.js'
+
+const QUERY_TYPE = 'score'
+
+export async function getScore(
+  pathToFetch,
+  client,
+  dates,
+  version = null,
+  verbose = false,
+  queryType = QUERY_TYPE,
+) {
+  const query = getScoreQuery(pathToFetch, dates, version)
+  const results = await runQuery(pathToFetch, query, client, queryType, verbose)
+  const data = JSON.parse(results.primaryResults[0].toString()).data[0]
+  // Extract Score
+  const score = data.Score
+  return score
+}
+
+export function getScoreQuery(pathToFetch, dates, version) {
+  return `
+    ${SHARED_DECLARATIONS(pathToFetch, dates, version)}
+    let _surveys = () {
+        docs_v0_survey_event
+    ${SHARED_FILTERS}
+    // Filter out Copilot response thumbs up/down events
+    | where context.event_group_key != "ask-ai"
+    // UNIQUE DO NOT DELETE
+    | summarize timestamp=arg_max(timestamp, *)
+        by User=toguid(context.user), Path=tostring(context.path_article)
+    };
+    _surveys
+    | summarize Score=round(
+        (countif(survey_vote) + 0.75 * 30) 
+          / (count() + 30), 
+        2
+      )
+    | project Score=strcat(toint(Score * 100), '%')
+  `
+}

src/metrics/queries/survey-score.js

@@ -0,0 +1,32 @@
+import { runQuery } from '#src/metrics/lib/kusto-client.js'
+import { SHARED_DECLARATIONS, SHARED_FILTERS } from '#src/metrics/queries/constants.js'
+
+const QUERY_TYPE = 'users'
+
+export async function getUsers(
+  pathToFetch,
+  client,
+  dates,
+  version = null,
+  verbose = false,
+  queryType = QUERY_TYPE,
+) {
+  const query = getUsersQuery(pathToFetch, dates, version)
+  const results = await runQuery(pathToFetch, query, client, queryType, verbose)
+  const data = JSON.parse(results.primaryResults[0].toString()).data[0]
+  // Extract Users
+  const users = data.Users
+  return users.toLocaleString()
+}
+
+export function getUsersQuery(pathToFetch, dates, version) {
+  return `
+    ${SHARED_DECLARATIONS(pathToFetch, dates, version)}
+    let _pages = () {
+        docs_v0_page_event
+    ${SHARED_FILTERS}
+    };
+    _pages
+    | summarize Users=dcount(tostring(context.user))
+  `
+}