diff --git a/apps/agent/tests/e2e/utils/report-UI-Tests-config.js b/apps/agent/tests/e2e/utils/report-UI-Tests-config.js new file mode 100644 index 0000000..6b7b6f8 --- /dev/null +++ b/apps/agent/tests/e2e/utils/report-UI-Tests-config.js @@ -0,0 +1,78 @@ +const { publish, defineConfig } = require("test-results-reporter"); +const dotenv = require("dotenv"); +const path = require("path"); +const fs = require("fs"); + +// Load .env from agent directory +// __dirname is apps/agent/tests/e2e/utils, go up 3 levels to apps/agent +dotenv.config({ path: path.resolve(__dirname, "../../../.env") }); + +const teamsHookBaseURL = process.env.DKG_Node_Teams_Hook; +const jenkinsUrl = process.env.JENKINS_URL; + +// Check if required environment variables are set +if (!teamsHookBaseURL) { + console.error("Error: DKG_Node_Teams_Hook environment variable is not set"); + console.error("Please add DKG_Node_Teams_Hook to your .env file in apps/agent/"); + process.exit(1); +} + +// Check if XML test results file exists +// __dirname is apps/agent/tests/e2e/utils, go up 3 levels to apps/agent +const xmlFilePath = path.resolve(__dirname, "../../../DKG_Node_UI_Tests.xml"); +if (!fs.existsSync(xmlFilePath)) { + console.error(`Error: Test results file not found at: ${xmlFilePath}`); + console.error("Please run the UI tests first using: npm run test:e2e"); + process.exit(1); +} + +// Build extensions array conditionally +const extensions = [ + { + name: "quick-chart-test-summary", + }, +]; + +// Only add hyperlinks extension if JENKINS_URL is set +if (jenkinsUrl) { + extensions.push({ + name: "hyperlinks", + inputs: { + links: [ + { + text: "UI Tests HTML Report", + url: `${jenkinsUrl}/job/DKG-Node-Tests/DKG_20Node_20UI_20Report/*zip*/DKG_20Node_20UI_20Report.zip`, + }, + ], + }, + }); +} + +const config = defineConfig({ + reports: [ + { + targets: [ + { + name: "teams", + condition: "fail", + inputs: { + url: teamsHookBaseURL, + only_failures: true, + publish: "test-summary-slim", + title: "DKG Node UI Tests Report", + width: "Full", + }, + extensions: extensions, + }, + ], + results: [ + { + type: "junit", + files: [xmlFilePath], + }, + ], + }, + ], +}); + +publish({ config }); diff --git a/apps/agent/tests/ragas/dashboard.ts b/apps/agent/tests/ragas/dashboard.ts index 6379934..cd445c7 100644 --- a/apps/agent/tests/ragas/dashboard.ts +++ b/apps/agent/tests/ragas/dashboard.ts @@ -38,9 +38,8 @@ interface EvaluationReport { } function getReportsDirectory(): string { - // Use absolute path to project root tests directory - const projectRoot = path.resolve(__dirname, "../../../../"); // Go to project root - return path.join(projectRoot, "tests/ragas/reports"); + // Reports are in the same directory structure as dashboard + return path.join(__dirname, "reports"); } function getAllReports(): any[] { diff --git a/apps/agent/tests/ragas/evaluate.ts b/apps/agent/tests/ragas/evaluate.ts index 3f03862..5f58e99 100644 --- a/apps/agent/tests/ragas/evaluate.ts +++ b/apps/agent/tests/ragas/evaluate.ts @@ -384,15 +384,21 @@ class DkgNodeRagasEvaluator { const scores = results.scores; const summary = this.generateSummary(scores); const recommendations = this.generateRecommendations(scores); - const failures = this.analyzeFailures(results.dataset, scores); + const detailedResults = results.detailedResults; + + const timestamp = new Date().toISOString(); + const timestampFormatted = timestamp.replace(/[:.]/g, "-"); const report = { + timestamp: timestamp, scores, summary, recommendations, - failures, - thresholds: this.config.thresholds, - timestamp: new Date().toISOString(), + detailedResults: detailedResults, + config: { + thresholds: this.config.thresholds, + metrics: this.config.metrics, + }, }; // Generate different report formats @@ -401,24 +407,29 @@ class DkgNodeRagasEvaluator { const dbJson = this.generateDatabaseJSON(scores); // Save reports to files - const timestamp = new Date().toISOString().replace(/[:.]/g, "-"); const reportsDir = path.join(__dirname, "reports"); + const evaluationDir = path.join(reportsDir, `evaluation-${timestampFormatted}`); - // Create reports directory if it doesn't exist - if (!fs.existsSync(reportsDir)) { - fs.mkdirSync(reportsDir, { recursive: true }); + // Create evaluation directory + if (!fs.existsSync(evaluationDir)) { + fs.mkdirSync(evaluationDir, { recursive: true }); } - // Save reports - const csvPath = path.join(reportsDir, `ragas-report-${timestamp}.csv`); + // Save main JSON report for dashboard + const jsonReportPath = path.join(evaluationDir, "evaluation-report.json"); + fs.writeFileSync(jsonReportPath, JSON.stringify(report, null, 2)); + + // Save additional formats + const csvPath = path.join(evaluationDir, `ragas-report-${timestampFormatted}.csv`); fs.writeFileSync(csvPath, csvReport); - const htmlPath = path.join(reportsDir, `ragas-report-${timestamp}.html`); + const htmlPath = path.join(evaluationDir, `ragas-report-${timestampFormatted}.html`); fs.writeFileSync(htmlPath, htmlReport); const dbJsonPath = path.join(__dirname, "ragas-results.json"); fs.writeFileSync(dbJsonPath, JSON.stringify(dbJson, null, 2)); + console.log(`\nšŸ“ Reports saved to: ${evaluationDir}`); console.log(`\nšŸŽÆ RAGAS Evaluation Summary:`); console.log( `Overall Score: ${(summary.overall.averageScore * 100).toFixed(1)}%`, diff --git a/apps/agent/tests/ragas/show-results.ts b/apps/agent/tests/ragas/show-results.ts index d7874cc..a839aaa 100644 --- a/apps/agent/tests/ragas/show-results.ts +++ b/apps/agent/tests/ragas/show-results.ts @@ -34,9 +34,8 @@ interface EvaluationReport { } function findLatestReport(): string | null { - // Use absolute path to project root tests directory - const projectRoot = path.resolve(__dirname, "../../../../"); // Go to project root - const reportsDir = path.join(projectRoot, "tests/ragas/reports"); + // Reports are in the same directory structure + const reportsDir = path.join(__dirname, "reports"); if (!fs.existsSync(reportsDir)) { console.log("šŸ“ No reports directory found"); diff --git a/package.json b/package.json index d0f7b31..2f6cbbc 100644 --- a/package.json +++ b/package.json @@ -15,9 +15,7 @@ "test:e2e": "turbo run test:e2e --filter=@dkg/agent --force", "test:api": "turbo run test --filter='@dkg/plugin-*' --force", "test:integration": "cd apps/agent && npm run test:integration", - "test:ui:report": "node report-UI-Tests-config.js", - "test:api:report": "node report-API-Tests-config.js", - "test:integration:report": "node report-Integration-Tests-config.js", + "test:ui:report": "node apps/agent/tests/e2e/utils/report-UI-Tests-config.js", "ragas:insert:guardian": "RAGAS_SOURCE=guardian node apps/agent/tests/ragas/scripts/insert_ragas_to_db.js", "ragas:insert:dkg_node": "RAGAS_SOURCE=dkg_node node apps/agent/tests/ragas/scripts/insert_ragas_to_db.js", "ragas": "cd apps/agent && npm run ragas"