feat: ingestion

README.md

@@ -50,6 +50,11 @@ hdcli tracker init
 hdcli tracker run
 ```
 
+```bash
+# Send information about your project manifest files
+hdcli report ingestion
+```
+
 ## Tutorials
 
 - [Configure your project to consume a Never-Ending Support package](docs/nes-init.md)

apps/cli/.eslintrc.json

@@ -27,6 +27,7 @@
               "graphql",
               "@herodevs/report-committers",
               "@herodevs/report-diagnostics",
+              "@herodevs/report-ingestion",
               "@herodevs/tracker-init",
               "@herodevs/tracker-run",
               "@herodevs/nes-init",

apps/cli/src/lib/get-commands.ts

@@ -2,6 +2,7 @@ import * as yargs from 'yargs';
 import { CommandModule } from 'yargs';
 import { reportCommittersCommand } from '@herodevs/report-committers';
 import { reportDiagnosticsCommand } from '@herodevs/report-diagnostics';
+import { reportIngestionCommand } from '@herodevs/report-ingestion';
 import { trackerInitCommand } from '@herodevs/tracker-init';
 import { trackerRunCommand } from '@herodevs/tracker-run';
 import { createGroupCommand } from './create-group-command';
@@ -25,7 +26,7 @@ export function getCommands(): CommandModule<any, any>[] {
     'type',
     'type of report',
     'r',
-    [reportCommittersCommand, reportDiagnosticsCommand],
+    [reportCommittersCommand, reportDiagnosticsCommand, reportIngestionCommand],
     'Invalid report type'
   );
 

libs/report/ingestion/.eslintrc.json

@@ -0,0 +1,25 @@
+{
+  "extends": ["../../../.eslintrc.base.json"],
+  "ignorePatterns": ["!**/*"],
+  "overrides": [
+    {
+      "files": ["*.ts", "*.tsx", "*.js", "*.jsx"],
+      "rules": {}
+    },
+    {
+      "files": ["*.ts", "*.tsx"],
+      "rules": {}
+    },
+    {
+      "files": ["*.js", "*.jsx"],
+      "rules": {}
+    },
+    {
+      "files": ["*.json"],
+      "parser": "jsonc-eslint-parser",
+      "rules": {
+        "@nx/dependency-checks": "error"
+      }
+    }
+  ]
+}

libs/report/ingestion/README.md

@@ -0,0 +1,11 @@
+# ingestion
+
+This library was generated with [Nx](https://nx.dev).
+
+## Building
+
+Run `nx build ingestion` to build the library.
+
+## Running unit tests
+
+Run `nx test ingestion` to execute the unit tests via [Jest](https://jestjs.io).

libs/report/ingestion/jest.config.ts

@@ -0,0 +1,11 @@
+/* eslint-disable */
+export default {
+  displayName: 'ingestion',
+  preset: '../../../jest.preset.js',
+  testEnvironment: 'node',
+  transform: {
+    '^.+\\.[tj]s$': ['ts-jest', { tsconfig: '<rootDir>/tsconfig.spec.json' }],
+  },
+  moduleFileExtensions: ['ts', 'js', 'html'],
+  coverageDirectory: '../../../coverage/libs/report/ingestion',
+};

libs/report/ingestion/package.json

@@ -0,0 +1,11 @@
+{
+  "name": "@herodevs/report-ingestion",
+  "version": "0.0.1",
+  "dependencies": {
+    "tslib": "^2.3.0"
+  },
+  "type": "commonjs",
+  "main": "./src/index.js",
+  "typings": "./src/index.d.ts",
+  "private": true
+}

libs/report/ingestion/project.json

@@ -0,0 +1,19 @@
+{
+  "name": "report-ingestion",
+  "$schema": "../../../node_modules/nx/schemas/project-schema.json",
+  "sourceRoot": "libs/report/ingestion/src",
+  "projectType": "library",
+  "tags": [],
+  "targets": {
+    "build": {
+      "executor": "@nx/js:tsc",
+      "outputs": ["{options.outputPath}"],
+      "options": {
+        "outputPath": "dist/libs/report/ingestion",
+        "main": "libs/report/ingestion/src/index.ts",
+        "tsConfig": "libs/report/ingestion/tsconfig.lib.json",
+        "assets": ["libs/report/ingestion/*.md"]
+      }
+    }
+  }
+}

libs/report/ingestion/src/index.ts

@@ -0,0 +1 @@
+export * from './lib/ingestion';

libs/report/ingestion/src/lib/ingestion-manifest.spec.ts

@@ -0,0 +1,68 @@
+import { readFileSync, statSync } from 'node:fs';
+import { promptToProceedUploadFile } from './prompts';
+import { findManifestFile } from './send-manifest';
+
+jest.mock('node:fs', () => ({
+  readFileSync: jest.fn(),
+  statSync: jest.fn(),
+}));
+
+jest.mock('./prompts', () => ({
+  promptToProceedUploadFile: jest.fn(),
+}));
+
+describe('Telemetry Functions', () => {
+  it('should find manifest files correctly', async () => {
+    const mockFileName = 'package.json';
+    const mockFileData = '{"name": "test package"}';
+    const mockFileStat = { size: 1024 };
+
+    (statSync as jest.Mock).mockReturnValue(mockFileStat);
+    (readFileSync as jest.Mock).mockReturnValue(mockFileData);
+    (promptToProceedUploadFile as jest.Mock).mockResolvedValue(true);
+
+    const result = await findManifestFile();
+
+    expect(result).toEqual({ name: mockFileName, data: mockFileData });
+    expect(promptToProceedUploadFile).toHaveBeenCalledWith(mockFileName);
+  });
+
+  it('should warn if manifest file is empty', async () => {
+    const mockFileName = 'package.json';
+    const mockFileStat = { size: 0 };
+    (statSync as jest.Mock).mockReturnValue(mockFileStat);
+    const consoleWarnSpy = jest.spyOn(console, 'warn').mockImplementation();
+    const result = await findManifestFile();
+
+    expect(result).toBeUndefined();
+    expect(consoleWarnSpy).toHaveBeenCalledWith(`File ${mockFileName} is empty`);
+    consoleWarnSpy.mockRestore();
+  });
+
+  it('should warn if manifest file is too large', async () => {
+    const mockFileStat = { size: 6e6 }; // 6MB file, larger than the 5MB max size
+    (statSync as jest.Mock).mockReturnValue(mockFileStat);
+    const consoleWarnSpy = jest.spyOn(console, 'warn').mockImplementation();
+    const result = await findManifestFile();
+    const mockFileName = 'package.json';
+    expect(result).toBeUndefined();
+    expect(consoleWarnSpy).toHaveBeenCalledWith(`File ${mockFileName} is too large`);
+    consoleWarnSpy.mockRestore();
+  });
+
+  it('should not proceed with upload if user rejects', async () => {
+    const mockFileName = 'package.json';
+    const mockFileStat = { size: 1024 };
+    (statSync as jest.Mock).mockReturnValue(mockFileStat);
+    (promptToProceedUploadFile as jest.Mock).mockResolvedValue(false);
+    const result = await findManifestFile();
+    expect(result).toBeUndefined();
+    expect(promptToProceedUploadFile).toHaveBeenCalledWith(mockFileName);
+  });
+
+  it('should return undefined if no manifest file is found', async () => {
+    (statSync as jest.Mock).mockReturnValueOnce(undefined);
+    const result = await findManifestFile();
+    expect(result).toBeUndefined();
+  });
+});

libs/report/ingestion/src/lib/ingestion-prompt.spec.ts

@@ -0,0 +1,95 @@
+import { input, confirm } from '@inquirer/prompts';
+import { askConsent, promptClientName, promptToProceedUploadFile } from './prompts';
+
+jest.mock('@inquirer/prompts', () => ({
+  input: jest.fn(),
+  confirm: jest.fn(),
+}));
+
+describe('askConsent', () => {
+  it('should return true if args.consent is true', async () => {
+    const args = { consent: true } as any;
+    const consoleSpy = jest.spyOn(console, 'log').mockImplementation();
+
+    const result = await askConsent(args);
+
+    expect(consoleSpy).toHaveBeenCalledWith(
+      'Data may contain sensitive data, please review before sharing it.'
+    );
+    expect(result).toBe(true);
+
+    consoleSpy.mockRestore();
+  });
+
+  it('should prompt for consent and return true if user agrees', async () => {
+    const args = { consent: false } as any;
+    (confirm as jest.Mock).mockResolvedValue(true);
+
+    const result = await askConsent(args);
+
+    expect(confirm).toHaveBeenCalledWith({
+      message: 'Data may contain sensitive data, please review before sharing it. Continue?',
+    });
+    expect(result).toBe(true);
+  });
+
+  it('should prompt for consent and return false if user disagrees', async () => {
+    const args = { consent: false } as any;
+    (confirm as jest.Mock).mockResolvedValue(false);
+
+    const result = await askConsent(args);
+
+    expect(confirm).toHaveBeenCalledWith({
+      message: 'Data may contain sensitive data, please review before sharing it. Continue?',
+    });
+    expect(result).toBe(false);
+  });
+});
+
+describe('promptClientName', () => {
+  it('should return the entered name if valid', async () => {
+    const mockName = 'John Doe';
+    (input as jest.Mock).mockResolvedValue(mockName);
+
+    const result = await promptClientName();
+
+    expect(input).toHaveBeenCalledWith({
+      message: 'Please enter your name:',
+      validate: expect.any(Function),
+    });
+    expect(result).toBe(mockName);
+  });
+
+  it('should validate the input and reject empty names', async () => {
+    const validateFn = (input as jest.Mock).mock.calls[0][0].validate;
+    expect(validateFn('')).toBe('Name cannot be empty!');
+    expect(validateFn('   ')).toBe('Name cannot be empty!');
+    expect(validateFn('Valid Name')).toBe(true);
+  });
+});
+
+describe('promptToProceedUploadFile', () => {
+  it('should return true if the user confirms the upload', async () => {
+    const fileName = 'test-file.txt';
+    (confirm as jest.Mock).mockResolvedValue(true);
+
+    const result = await promptToProceedUploadFile(fileName);
+
+    expect(result).toBe(true);
+    expect(confirm).toHaveBeenCalledWith({
+      message: `Found ${fileName}, this file will be uploaded. Continue?`,
+    });
+  });
+
+  it('should return false if the user denies the upload', async () => {
+    const fileName = 'test-file.txt';
+    (confirm as jest.Mock).mockResolvedValue(false);
+
+    const result = await promptToProceedUploadFile(fileName);
+
+    expect(result).toBe(false);
+    expect(confirm).toHaveBeenCalledWith({
+      message: `Found ${fileName}, this file will be uploaded. Continue?`,
+    });
+  });
+});

libs/report/ingestion/src/lib/ingestion.ts

@@ -0,0 +1,39 @@
+import { type ArgumentsCamelCase, type CommandModule } from 'yargs';
+import { askConsent, promptClientName, promptToProceedUploadFile } from './prompts';
+import { findManifestFile, getClientToken, sendManifest } from './send-manifest';
+import { type Options } from './types';
+
+export const reportIngestionCommand: CommandModule<object, Options> = {
+  command: 'ingestion',
+  describe: 'send manifest files information',
+  aliases: ['ingest', 'i'],
+  builder: {
+    consent: {
+      describe: 'Agree to understanding that sensitive data may be outputted',
+      required: false,
+      default: false,
+      boolean: true,
+    },
+  },
+  handler: run,
+};
+
+async function run(args: ArgumentsCamelCase<Options>): Promise<void> {
+  const consent = await askConsent(args);
+  if (!consent) {
+    return;
+  }
+  // Prompt the user to insert their name
+  const clientName = await promptClientName();
+  // First we need to get a short lived token
+  const oid = await getClientToken(clientName);
+
+  const manifest = await findManifestFile();
+  if (!manifest) {
+    console.log('No manifest files found');
+    return;
+  }
+
+  await sendManifest(oid, manifest, { clientName });
+  console.log('Manifest sent correctly!');
+}

libs/report/ingestion/src/lib/prompts.ts

@@ -0,0 +1,37 @@
+import { confirm, input } from '@inquirer/prompts';
+import { ArgumentsCamelCase } from 'yargs';
+import { type Options } from './types';
+
+export async function askConsent(args: ArgumentsCamelCase<Options>): Promise<boolean> {
+  const consentPrompt = 'Data may contain sensitive data, please review before sharing it.';
+  if (!args.consent) {
+    const answer = await confirm({
+      message: `${consentPrompt} Continue?`,
+    });
+    if (!answer) {
+      return false;
+    }
+  } else {
+    console.log(consentPrompt);
+  }
+  return true;
+}
+
+export async function promptClientName() {
+  const name = await input({
+    message: 'Please enter your name:',
+    validate: (value) => (value.trim() === '' ? 'Name cannot be empty!' : true),
+  });
+  return name;
+}
+
+export async function promptToProceedUploadFile(fileName: string): Promise<boolean> {
+  const consentPrompt = `Found ${fileName}, this file will be uploaded.`;
+  const answer = await confirm({
+    message: `${consentPrompt} Continue?`,
+  });
+  if (!answer) {
+    return false;
+  }
+  return true;
+}

libs/report/ingestion/src/lib/queries.ts

@@ -0,0 +1,26 @@
+import { gql } from '@apollo/client/core';
+
+export const TELEMETRY_INITIALIZE_MUTATION = gql`
+  mutation Telemetry($clientName: String!) {
+    telemetry {
+      initialize(input: { context: { client: { id: $clientName } } }) {
+        success
+        oid
+        message
+      }
+    }
+  }
+`;
+
+export const TELEMETRY_REPORT_MUTATION = gql`
+  mutation Report($key: String!, $report: JSON!, $metadata: JSON) {
+    telemetry {
+      report(input: { key: $key, report: $report, metadata: $metadata }) {
+        txId
+        success
+        message
+        diagnostics
+      }
+    }
+  }
+`;