Add ability to sort extensions by downloads, using stats pulled from Tableau

Author: holly-cummins

.github/workflows/build_and_publish.yml

@@ -62,6 +62,8 @@ jobs:
           NODE_ENV: production
           GATSBY_ACTIVE_ENV: production
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          TABLEAU_PERSONAL_ACCESS_TOKEN: ${{ secrets.TABLEAU_PERSONAL_ACCESS_TOKEN }}
+          TABLEAU_SITE: ${{ secrets.TABLEAU_SITE }}
           SEGMENT_KEY: ${{ secrets.SEGMENT_KEY }}
 
       - name: Caching GitHub API results

README.md

@@ -27,8 +27,15 @@ nvm use 18
 
 ## Environment variables 
 
-Information is more complete if a GitHub access token is provided. It should only be granted read access. 
-Set it as an environment variable called `GITHUB_TOKEN`. (In the CI, this will be provided by the platform.)
+The site pulls data from a range of sources, some of which need credentials. For the full build, set the following environment variables:
+
+- `GITHUB_TOKEN` (this will be automatically set in a GitHub CI, and should only be granted read access)
+- `TABLEAU_PERSONAL_ACCESS_TOKEN`
+- `TABLEAU_SITE`
+- `SEGMENT_KEY` (used for anonymised analytics)
+
+Information is more complete if a these tokens are provided, but the build should still succeed if they are missing. If it fails without them, please raise an issue.
+In PR builds, everything except the `GITHUB_TOKEN` will be missing.
 
 ## Caching 
 

gatsby-config.js

@@ -87,6 +87,7 @@ module.exports = {
         },
       },
     },
+    "download-data",
     {
       resolve: `gatsby-plugin-feed`,
       options: {

gatsby-node.js

@@ -202,6 +202,11 @@ exports.sourceNodes = async ({
 
     }
 
+    if (node.metadata) {
+      // Do the link to the download data
+      node.metadata.downloads = node.metadata?.maven?.artifactId
+    }
+
     return createNode(node)
   })
   return Promise.all(secondPromises)
@@ -337,6 +342,7 @@ exports.createSchemaCustomization = ({ actions }) => {
       icon: File @link(by: "url")
       sponsors: [String]
       sponsor: String
+      downloads: DownloadRanking @link(by: "artifactId")
     }
     
     type MavenInfo {

package-lock.json

@@ -15,6 +15,7 @@
     "axios": "^1.6.7",
     "cacache": "^18.0.1",
     "compare-version": "^0.1.2",
+    "csvtojson": "^2.0.10",
     "date-fns": "^3.3.1",
     "encodeurl": "^1.0.2",
     "eslint-plugin-gatsby": "^1.0.2",

package.json

@@ -0,0 +1,44 @@
+const { getMostRecentData } = require("./tableau-fetcher")
+
+const type = "DownloadRanking"
+
+exports.sourceNodes = async ({ actions, createNodeId, createContentDigest }) => {
+  const { createNode } = actions
+
+  const allData = await getMostRecentData()
+
+  if (allData) {
+    
+    createNode({
+      date: allData.date,
+      id: createNodeId(allData.date.toString()),
+      internal: { type: "DownloadDataDate", contentDigest: createContentDigest(allData.date) }
+    })
+
+    const promises = allData.ranking.map(artifact =>
+      createNode({
+        ...artifact,
+        id: createNodeId(artifact.artifactId),
+        internal: { type, contentDigest: createContentDigest(artifact.artifactId + artifact.rank) }
+      })
+    )
+    // Return a promise to make sure we wait
+    return Promise.all(promises)
+  }
+}
+
+exports.createSchemaCustomization = ({ actions }) => {
+  const { createTypes } = actions
+  const typeDefs = `
+  type DownloadRanking implements Node {
+    artifactId: String
+    rank: Int
+  }
+  
+  type DownloadDataDate implements Node {
+    date: String
+  }
+  `
+  createTypes(typeDefs)
+
+}

plugins/download-data/gatsby-node.js

@@ -0,0 +1,54 @@
+/**
+ * @jest-environment node
+ */
+
+const { sourceNodes } = require("./gatsby-node")
+
+const dataFetcher = require("./tableau-fetcher")
+
+jest.mock("./tableau-fetcher")
+
+const stats = {
+  date: new Date(),
+  ranking: [
+    { artifactId: "quarkus-popular", rank: 1 },
+    { artifactId: "quarkus-soso", rank: 2 },
+    { artifactId: "tools", rank: 3 }]
+}
+dataFetcher.getMostRecentData.mockResolvedValue(stats)
+
+const contentDigest = "some content digest"
+const createNode = jest.fn()
+const createNodeId = jest.fn()
+const createContentDigest = jest.fn().mockReturnValue(contentDigest)
+const actions = { createNode }
+
+describe("the download data supplier", () => {
+
+
+  beforeAll(async () => {
+    await sourceNodes({ createContentDigest, createNodeId, actions })
+  })
+
+  it("gets the download data", async () => {
+    expect(dataFetcher.getMostRecentData).toHaveBeenCalled()
+  })
+
+  it("creates a new node for each artifact", async () => {
+    expect(createNode).toHaveBeenCalledWith(
+      expect.objectContaining({ artifactId: "tools", rank: 3 })
+    )
+    expect(createNode).toHaveBeenCalledWith(
+      expect.objectContaining({ artifactId: "quarkus-popular", rank: 1 },
+      )
+    )
+  })
+
+  it("adds gatsby metadata to the nodes", async () => {
+    const type = "DownloadRanking"
+    expect(createNode).toHaveBeenCalledWith(expect.objectContaining({
+      internal: { type, contentDigest: expect.anything() }
+    }))
+  })
+
+})

plugins/download-data/gatsby-node.test.js

@@ -0,0 +1,3 @@
+{
+  "name": "download-data"
+}

plugins/download-data/package.json

@@ -0,0 +1,121 @@
+const axios = require("axios")
+const csv = require("csvtojson")
+
+const serverUrl = "https://10ay.online.tableau.com/api/3.22"
+const site = process.env["TABLEAU_SITE"]
+
+// https://help.tableau.com/current/api/rest_api/en-us/REST/rest_api_get_started_tutorial_part_1.htm is the useful docs for this
+async function getAccessToken() {
+  const personalAccessToken = process.env["TABLEAU_PERSONAL_ACCESS_TOKEN"]
+
+  if (personalAccessToken) {
+    const tokenUrl = `${serverUrl}/auth/signin`
+
+    const xml = `<tsRequest>
+    <credentials 
+    personalAccessTokenName="extensions-site"
+    personalAccessTokenSecret="${personalAccessToken}" >
+      <site contentUrl="${site}" />
+    </credentials>
+  </tsRequest>`
+
+    const response = await axios.post(tokenUrl, xml)
+
+    return { token: response.data.credentials.token, siteId: response.data.credentials.site.id }
+  } else {
+    console.log("No TABLEAU_PERSONAL_ACCESS_TOKEN has been set. Not fetching download data.")
+  }
+
+}
+
+async function getViewId(accessToken, siteId) {
+  // This can almost be read off the web page, but note the extra 'sheets' in the middle
+  // https://help.tableau.com/current/api/rest_api/en-us/REST/rest_api_concepts_filtering_and_sorting.htm#filter-expressions has guidance on other ways of finding the right view
+  // Other fields we could use are
+  //       name: 'Downloads Highlight table',
+  //       contentUrl: 'Quark-us-iverse-MavenNexusCommunityDownloads/sheets/DownloadsHighlighttable',
+  //       viewUrlName: 'DownloadsHighlighttable'
+  //     }
+  const contentUrl = "Quark-us-iverse-MavenNexusCommunityDownloads/sheets/DownloadsHighlighttable"
+  const downloadUrl = `${serverUrl}/sites/${siteId}/views?filter=contentUrl:eq:${contentUrl}`
+
+  const response = await axios.get(downloadUrl, {
+    headers: {
+      "X-Tableau-Auth": accessToken
+    }
+  })
+
+  if (!response.data?.views?.view[0]) {
+    console.error("Could not find the view with content url", contentUrl)
+  }
+  return response.data.views.view[0].id
+}
+
+async function downloadViewAsCsv(accessToken, siteId, viewId) {
+  const downloadUrl = `${serverUrl}/sites/${siteId}/views/${viewId}/data`
+
+  const response = await axios.get(downloadUrl, {
+    headers: {
+      "X-Tableau-Auth": accessToken
+    }
+  })
+
+  return response.data
+}
+
+const getCsv = async () => {
+  const tokenData = await getAccessToken()
+
+  if (tokenData) {
+    const { token, siteId } = tokenData
+    const viewId = await getViewId(token, siteId)
+    return await downloadViewAsCsv(token, siteId, viewId)
+  }
+}
+
+const convertToNumber = (s) => {
+  // We can't call parseInt directly, because Tableau gives us comma-separated strings, and parse int, despite the name, can't handle them.
+  return s ? parseInt(s.replaceAll(",", "")) : -1
+}
+
+const getMostRecentData = async () => {
+
+  const csvData = await getCsv()
+
+  if (csvData) {
+    const json = await csv({
+      noheader: false,
+      flatKeys: true
+    })
+      .fromString(csvData)
+
+    // Normalise the headers
+    // We are expecting data.artifactId,Month of Data.Date,Year of Data.Date,Data.Timeline
+    const withDates = json
+      .map(entry => {
+        return {
+          artifactId: entry["data.artifactId"],
+          month: entry["Month of Data.Date"],
+          downloads: convertToNumber(entry["Data.Timeline"])
+        }
+      })
+      .map(entry => {
+        return { date: new Date(entry.month), ...entry }
+      })
+
+    const mostRecentDate = withDates.map(entry => entry.date).filter(date => date.getTime() > 0).sort((a, b) => b - a)[0]
+
+    const onlyMostRecentDownloads = withDates.filter(entry => entry.date.getTime() === mostRecentDate.getTime())
+      .sort((a, b) => b.downloads - a.downloads)
+
+    const ranking = onlyMostRecentDownloads.map((entry, i) => {
+      return { artifactId: entry.artifactId, rank: i + 1 }
+    })
+
+    return { date: mostRecentDate, ranking }
+  }
+
+}
+
+// Exported for testing
+module.exports = { getCsv, getMostRecentData }