Add auto approve based on history (#95)

pashidlos · web-flow · commit 58ea1a3a3ae0 · 2021-01-30T14:58:09.000+02:00
* Add auto approve based on history Visual-Regression-Tracker/Visual-Regression-Tracker#190
diff --git a/.env b/.env
@@ -14,4 +14,7 @@ POSTGRES_DB=vrt_db_dev
 
 # optional
 #HTTPS_KEY_PATH='./secrets/ssl.key'
-#HTTPS_CERT_PATH='./secrets/ssl.cert'
+#HTTPS_CERT_PATH='./secrets/ssl.cert'
+
+# features
+AUTO_APPROVE_BASED_ON_HISTORY=true
diff --git a/prisma/migrations/20210130115922-test-run-auto-approve-status-added/README.md b/prisma/migrations/20210130115922-test-run-auto-approve-status-added/README.md
@@ -0,0 +1,37 @@
+# Migration `20210130115922-test-run-auto-approve-status-added`
+
+This migration has been generated by Pavlo Strunkin at 1/30/2021, 1:59:22 PM.
+You can check out the [state of the schema](./schema.prisma) after the migration.
+
+## Database Steps
+
+```sql
+ALTER TYPE "TestStatus" ADD VALUE 'autoApproved'
+```
+
+## Changes
+
+```diff
+diff --git schema.prisma schema.prisma
+migration 20210118201534-build--project-id---ci-build-id-constraint..20210130115922-test-run-auto-approve-status-added
+--- datamodel.dml
++++ datamodel.dml
+@@ -3,9 +3,9 @@
+ }
+ datasource db {
+   provider = "postgresql"
+-  url = "***"
++  url = "***"
+ }
+ model Build {
+   id         String    @id @default(uuid())
+@@ -118,5 +118,6 @@
+   new
+   ok
+   unresolved
+   approved
++  autoApproved
+ }
+```
+
+
diff --git a/prisma/migrations/20210130115922-test-run-auto-approve-status-added/schema.prisma b/prisma/migrations/20210130115922-test-run-auto-approve-status-added/schema.prisma
@@ -0,0 +1,123 @@
+generator client {
+  provider = "prisma-client-js"
+}
+
+datasource db {
+  provider = "postgresql"
+  url = "***"
+}
+
+model Build {
+  id         String    @id @default(uuid())
+  ciBuildId  String?
+  number     Int?
+  branchName String?
+  status     String?
+  testRuns   TestRun[]
+  projectId  String
+  project    Project   @relation(fields: [projectId], references: [id])
+  updatedAt  DateTime  @updatedAt
+  createdAt  DateTime  @default(now())
+  user       User?     @relation(fields: [userId], references: [id])
+  userId     String?
+  isRunning  Boolean?
+
+  @@unique([projectId, ciBuildId])
+}
+
+model Project {
+  id             String          @id @default(uuid())
+  name           String
+  mainBranchName String          @default("master")
+  builds         Build[]
+  buildsCounter  Int             @default(0)
+  testVariations TestVariation[]
+  updatedAt      DateTime        @updatedAt
+  createdAt      DateTime        @default(now())
+
+  @@unique([name])
+}
+
+model TestRun {
+  id                    String        @id @default(uuid())
+  imageName             String
+  diffName              String?
+  diffPercent           Float?
+  diffTollerancePercent Float         @default(0)
+  pixelMisMatchCount    Int?
+  status                TestStatus
+  buildId               String
+  build                 Build         @relation(fields: [buildId], references: [id])
+  testVariationId       String
+  testVariation         TestVariation @relation(fields: [testVariationId], references: [id])
+  merge                 Boolean       @default(false)
+  updatedAt             DateTime      @updatedAt
+  createdAt             DateTime      @default(now())
+  // Test variation data
+  name                  String        @default("")
+  browser               String?
+  device                String?
+  os                    String?
+  viewport              String?
+  baselineName          String?
+  comment               String?
+  baseline              Baseline?
+  branchName            String        @default("master")
+  baselineBranchName    String?
+  ignoreAreas           String        @default("[]")
+  tempIgnoreAreas       String        @default("[]")
+}
+
+model TestVariation {
+  id           String     @id @default(uuid())
+  name         String
+  branchName   String     @default("master")
+  browser      String?
+  device       String?
+  os           String?
+  viewport     String?
+  baselineName String?
+  ignoreAreas  String     @default("[]")
+  projectId    String
+  project      Project    @relation(fields: [projectId], references: [id])
+  testRuns     TestRun[]
+  baselines    Baseline[]
+  comment      String?
+  updatedAt    DateTime   @updatedAt
+  createdAt    DateTime   @default(now())
+
+  @@unique([projectId, name, browser, device, os, viewport, branchName])
+}
+
+model Baseline {
+  id              String        @id @default(uuid())
+  baselineName    String
+  testVariationId String
+  testVariation   TestVariation @relation(fields: [testVariationId], references: [id])
+  testRunId       String?
+  testRun         TestRun?      @relation(fields: [testRunId], references: [id])
+  updatedAt       DateTime      @updatedAt
+  createdAt       DateTime      @default(now())
+}
+
+model User {
+  id        String   @id @default(uuid())
+  email     String   @unique
+  password  String
+  firstName String?
+  lastName  String?
+  apiKey    String   @unique
+  isActive  Boolean  @default(true)
+  builds    Build[]
+  updatedAt DateTime @updatedAt
+  createdAt DateTime @default(now())
+}
+
+enum TestStatus {
+  failed
+  new
+  ok
+  unresolved
+  approved
+  autoApproved
+}
diff --git a/prisma/migrations/20210130115922-test-run-auto-approve-status-added/steps.json b/prisma/migrations/20210130115922-test-run-auto-approve-status-added/steps.json
@@ -0,0 +1,12 @@
+{
+  "version": "0.3.14-fixed",
+  "steps": [
+    {
+      "tag": "UpdateEnum",
+      "enum": "TestStatus",
+      "createdValues": [
+        "autoApproved"
+      ]
+    }
+  ]
+}
diff --git a/prisma/migrations/migrate.lock b/prisma/migrations/migrate.lock
@@ -12,4 +12,5 @@
 20201007145002-builds-counter
 20201115155739-ci-build-id-added
 20201201211711-test-run--temp-ignore-areas-added
-20210118201534-build--project-id---ci-build-id-constraint
+20210118201534-build--project-id---ci-build-id-constraint
+20210130115922-test-run-auto-approve-status-added
diff --git a/prisma/schema.prisma b/prisma/schema.prisma
@@ -119,4 +119,5 @@ enum TestStatus {
   ok
   unresolved
   approved
+  autoApproved
 }
diff --git a/src/test-runs/test-runs.service.spec.ts b/src/test-runs/test-runs.service.spec.ts
@@ -514,6 +514,8 @@ describe('TestRunsService', () => {
     service.getDiff = getDiffMock;
     const saveDiffResultMock = jest.fn();
     service.saveDiffResult = saveDiffResultMock.mockResolvedValueOnce(testRunWithResult);
+    const tryAutoApproveBasedOnHistory = jest.fn();
+    service['tryAutoApproveBasedOnHistory'] = tryAutoApproveBasedOnHistory.mockResolvedValueOnce(testRunWithResult);
 
     const result = await service.create(testVariation, createTestRequestDto);
 
@@ -564,6 +566,20 @@ describe('TestRunsService', () => {
       },
     ]);
     expect(saveDiffResultMock).toHaveBeenCalledWith(testRun.id, diffResult);
+    expect(tryAutoApproveBasedOnHistory).toHaveBeenCalledWith(testVariation, testRunWithResult, image, [
+      {
+        x: 3,
+        y: 4,
+        width: 500,
+        height: 600,
+      },
+      {
+        x: 1,
+        y: 2,
+        width: 100,
+        height: 200,
+      },
+    ]);
     expect(eventTestRunCreatedMock).toHaveBeenCalledWith(testRunWithResult);
     expect(result).toBe(testRunWithResult);
   });
diff --git a/src/test-runs/test-runs.service.ts b/src/test-runs/test-runs.service.ts
@@ -1,4 +1,4 @@
-import { forwardRef, Inject, Injectable } from '@nestjs/common';
+import { forwardRef, Inject, Injectable, Logger } from '@nestjs/common';
 import { PNG } from 'pngjs';
 import Pixelmatch from 'pixelmatch';
 import { CreateTestRequestDto } from './dto/create-test-request.dto';
@@ -14,9 +14,12 @@ import { TestVariationsService } from '../test-variations/test-variations.servic
 import { convertBaselineDataToQuery } from '../shared/dto/baseline-data.dto';
 import { TestRunDto } from './dto/testRun.dto';
 import { PaginatedTestRunDto } from './dto/testRun-paginated.dto';
+import { getTestVariationUniqueData } from '../utils';
 
 @Injectable()
 export class TestRunsService {
+  private readonly logger: Logger = new Logger(TestRunsService.name);
+
   constructor(
     @Inject(forwardRef(() => TestVariationsService))
     private testVariationService: TestVariationsService,
@@ -81,7 +84,8 @@ export class TestRunsService {
     return new TestRunResultDto(testRun, testVariation);
   }
 
-  async approve(id: string, merge: boolean): Promise<TestRun> {
+  async approve(id: string, merge: boolean, autoApprove?: boolean): Promise<TestRun> {
+    const status = autoApprove ? TestStatus.autoApproved : TestStatus.approved;
     const testRun = await this.findOne(id);
 
     // save new baseline
@@ -92,7 +96,7 @@ export class TestRunsService {
       testRunUpdated = await this.prismaService.testRun.update({
         where: { id },
         data: {
-          status: TestStatus.approved,
+          status,
           testVariation: {
             update: {
               baselineName,
@@ -115,11 +119,7 @@ export class TestRunsService {
         data: {
           project: { connect: { id: testRun.testVariation.projectId } },
           baselineName,
-          name: testRun.name,
-          browser: testRun.browser,
-          device: testRun.device,
-          os: testRun.os,
-          viewport: testRun.viewport,
+          ...getTestVariationUniqueData(testRun),
           ignoreAreas: testRun.ignoreAreas,
           comment: testRun.comment,
           branchName: testRun.branchName,
@@ -141,7 +141,7 @@ export class TestRunsService {
       testRunUpdated = await this.prismaService.testRun.update({
         where: { id },
         data: {
-          status: TestStatus.approved,
+          status,
           testVariation: {
             connect: { id: newTestVariation.id },
           },
@@ -209,11 +209,7 @@ export class TestRunsService {
             id: createTestRequestDto.buildId,
           },
         },
-        name: testVariation.name,
-        browser: testVariation.browser,
-        device: testVariation.device,
-        os: testVariation.os,
-        viewport: testVariation.viewport,
+        ...getTestVariationUniqueData(testVariation),
         baselineName: testVariation.baselineName,
         baselineBranchName: testVariation.branchName,
         ignoreAreas: testVariation.ignoreAreas,
@@ -235,7 +231,10 @@ export class TestRunsService {
     }
     const diffResult = this.getDiff(baseline, image, testRun.diffTollerancePercent, ignoreAreas);
 
-    const testRunWithResult = await this.saveDiffResult(testRun.id, diffResult);
+    let testRunWithResult = await this.saveDiffResult(testRun.id, diffResult);
+
+    testRunWithResult = await this.tryAutoApproveBasedOnHistory(testVariation, testRunWithResult, image, ignoreAreas);
+
     this.eventsGateway.testRunCreated(testRunWithResult);
     return testRunWithResult;
   }
@@ -334,4 +333,70 @@ export class TestRunsService {
     });
     return image.data;
   }
+
+  private async tryAutoApproveBasedOnHistory(
+    testVariation: TestVariation,
+    testRun: TestRun,
+    image: PNG,
+    ignoreAreas: IgnoreAreaDto[]
+  ): Promise<TestRun> {
+    if (process.env.AUTO_APPROVE_BASED_ON_HISTORY && testRun.status !== TestStatus.ok) {
+      this.logger.log(`Try auto approve testRun: ${testRun.id}`);
+
+      const alreadyApprovedTestRuns: TestRun[] = await this.prismaService.testRun.findMany({
+        where: {
+          ...getTestVariationUniqueData(testVariation),
+          baselineName: testVariation.baselineName,
+          status: TestStatus.approved,
+        },
+      });
+
+      let autoApproved = false;
+      for (const approvedTestRun of alreadyApprovedTestRuns) {
+        this.logger.log(
+          `Found already approved baseline for testRun: ${testRun.id}
+        testVariation: ${approvedTestRun.testVariationId}
+        branch: ${approvedTestRun.branchName}
+        testRun: ${approvedTestRun.id}
+        build: ${approvedTestRun.buildId}`
+        );
+
+        const approvedTestVariation = await this.prismaService.testVariation.findUnique({
+          where: {
+            id: approvedTestRun.testVariationId,
+          },
+        });
+
+        const approvedBaseline = this.staticService.getImage(approvedTestVariation.baselineName);
+        const diffResult = this.getDiff(approvedBaseline, image, testRun.diffTollerancePercent, ignoreAreas);
+
+        if (diffResult.status === TestStatus.ok) {
+          autoApproved = true;
+          const baseline = await this.prismaService.baseline.findFirst({
+            where: {
+              testVariationId: approvedTestVariation.id,
+              baselineName: approvedTestVariation.baselineName,
+            },
+            include: {
+              testRun: true,
+            },
+          });
+          this.logger.log(
+            `Found reason to auto approve testRun: ${testRun.id}
+          testVariation: ${baseline.testVariationId}
+          baseline: ${baseline.id}
+          branch: ${approvedTestVariation.branchName}
+          testRun: ${baseline.testRunId}
+          build: ${baseline.testRun.buildId}`
+          );
+        }
+      }
+
+      if (autoApproved) {
+        return this.approve(testRun.id, false, true);
+      }
+      this.logger.log(`Cannot auto approve testRun: ${testRun.id}`);
+    }
+    return testRun;
+  }
 }
diff --git a/src/test-variations/test-variations.service.ts b/src/test-variations/test-variations.service.ts
diff --git a/src/utils/index.ts b/src/utils/index.ts

Original file line number	Diff line number	Diff line change
`@@ -119,4 +119,5 @@ enum TestStatus {`
`119`	`119`	`ok`
`120`	`120`	`unresolved`
`121`	`121`	`approved`
	`122`	`+ autoApproved`
`122`	`123`	`}`