Remove unused conda data-fetcher function and therefore remove DuckDB dependency. (#106)

* Can actually remove the whole function and the dependency!?

* And now these are unused.

* Regenerate package-lock.json.

---------

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

Author: samcunliffe

backend/package.json

@@ -58,7 +58,6 @@
     "esno": "^4.8.0",
     "fs-extra": "^11.3.3",
     "octokit": "^5.0.5",
-    "yaml": "^2.8.2",
-    "duckdb-async": "^1.4.2"
+    "yaml": "^2.8.2"
   }
 }

backend/src/fetchers/fetch_parquet.ts

@@ -1,12 +1,5 @@
 import fs from 'fs';
-import os from 'os';
-import path from 'path';
 import * as https from 'node:https';
-import { Config, Result } from '../index';
-import { CustomOctokit } from '../lib/octokit';
-import { queryRepoNames } from './fetcher_utils';
-
-import { Database } from 'duckdb-async';
 
 async function downloadParquetFile(url: string, outputPath: string) {
     return new Promise((resolve, reject) => {
@@ -25,81 +18,3 @@ async function downloadParquetFile(url: string, outputPath: string) {
         });
     });
 }
-
-export const addCondaData = async (result: Result, octokit: CustomOctokit, config: Config, startYear: number=2018) => {
-    const repos = await queryRepoNames(octokit, config);
-    const baseDir = path.join(os.homedir(), '.dashboard');
-
-    const packages = repos.map((repo) => {return repo.name });
-
-    if (!fs.existsSync(baseDir)) {
-        fs.mkdirSync(baseDir);
-    }
-
-    let currYear = new Date().getFullYear();
-    let lastMonth = 1;
-
-    for (let i = startYear; i <= currYear; i++) {
-        for (let j = 1; j <= 12; j++) {
-            const fileName = path.join(baseDir, `${i}-${String(j).padStart(2, '0')}.parquet`);
-
-            if (!fs.existsSync(fileName)) {
-                const url = `https://anaconda-package-data.s3.amazonaws.com/conda/monthly/${i}/${i}-${String(j).padStart(2, '0')}.parquet`;
-
-                // Check if the URL for the given month exists (status code 2xx)
-                // The updates can take some time to be available, so there's
-                // no easy way to know which month to stop at.
-                const checkURLReq = await new Promise((resolve, reject) => {
-                    fetch(url, {
-                        method: "HEAD"
-                    }).then(response => {
-                        resolve(response.status.toString()[0] === "2")
-                    }).catch(error => {
-                        reject(false)
-                    })
-                })
-
-                // If the URL does not exist, assume that there are no more
-                // files to download and break the loop
-                if (!checkURLReq) {
-                    lastMonth = j > 1 ? j - 1 : 12;
-                    // Make sure we break out of the outer loop as well
-                    // Update teh current year to the last year we downloaded
-                    currYear = i;
-                    break;
-                } else {
-                    await downloadParquetFile(url, fileName);
-                }
-            }
-        }
-    }
-
-    const db = await Database.create( `:memory:` );
-    const formattedString = packages.map((pkg) => `'${pkg}'`).join(',');
-
-    const totalDownloads = await db.all(`SELECT pkg_name, SUM(counts)::INTEGER AS total FROM '${baseDir}/*.parquet' WHERE pkg_name IN (${formattedString}) GROUP BY pkg_name`);
-
-    if (lastMonth == 12) {
-        currYear -= 1;
-    }
-
-    const lastMonthDownloads = await db.all(`SELECT pkg_name, SUM(counts)::INTEGER AS total FROM '${baseDir}/${currYear}-${String(lastMonth).padStart(2, '0')}.parquet' WHERE pkg_name IN (${formattedString}) GROUP BY pkg_name`);
-
-    totalDownloads.forEach((row) => {
-        if ( !result.repositories[row.pkg_name].condaTotalDownloads ) {
-            result.repositories[row.pkg_name].condaTotalDownloads = row.total;
-        } else {
-            result.repositories[row.pkg_name].condaTotalDownloads += row.total;
-        }
-    })
-
-    lastMonthDownloads.forEach((row) => {
-        if (!result.repositories[row.pkg_name].condaMonthlyDownloads) {
-            result.repositories[row.pkg_name].condaMonthlyDownloads = row.total;
-        } else {
-            result.repositories[row.pkg_name].condaMonthlyDownloads += row.total;
-        }
-    })
-
-    return result;
-};

backend/src/index.ts

@@ -13,7 +13,6 @@ import {
   addRepositoriesToResult,
 } from './fetchers';
 import { CustomOctokit, checkRateLimit, personalOctokit } from './lib/octokit';
-import { addCondaData } from './fetchers/fetch_parquet';
 
 export interface Result {
   meta: {
@@ -133,8 +132,7 @@ for (const orgName of configOrganizationName) {
     addRepositoriesToResult,
     addIssueAndPrData,
     addDiscussionData,
-    addIssueMetricsData,
-    addCondaData
+    addIssueMetricsData
   );
 
   outputResult(result, orgName);