#RI-5750 - update rdi with new api

Author: AmirAllayarovSofteq

redisinsight/api/src/modules/rdi/client/api.rdi.client.ts

@@ -14,10 +14,10 @@ import {
 import {
   RdiDryRunJobDto,
   RdiDryRunJobResponseDto,
+  RdiTemplateResponseDto,
   RdiTestConnectionsResponseDto,
 } from 'src/modules/rdi/dto';
 import {
-  RdiPipelineDeployFailedException,
   RdiPipelineInternalServerErrorException,
   wrapRdiPipelineError,
 } from 'src/modules/rdi/exceptions';
@@ -29,8 +29,7 @@ import {
 } from 'src/modules/rdi/models';
 import { convertKeysToCamelCase } from 'src/utils/base.helper';
 import { RdiPipelineTimeoutException } from 'src/modules/rdi/exceptions/rdi-pipeline.timeout-error.exception';
-
-const RDI_DEPLOY_FAILED_STATUS = 'failed';
+import * as https from 'https';
 
 export class ApiRdiClient extends RdiClient {
   protected readonly client: AxiosInstance;
@@ -42,6 +41,9 @@ export class ApiRdiClient extends RdiClient {
     this.client = axios.create({
       baseURL: rdi.url,
       timeout: RDI_TIMEOUT,
+      httpsAgent: new https.Agent({
+        rejectUnauthorized: false,
+      }),
     });
   }
 
@@ -72,25 +74,32 @@ export class ApiRdiClient extends RdiClient {
     }
   }
 
-  async getTemplate(options: object): Promise<object> {
+  async getConfigTemplate(pipelineType: string, dbType: string): Promise<RdiTemplateResponseDto> {
     try {
-      const response = await this.client.get(RdiUrl.GetTemplate, { params: options });
+      const response = await this.client.get(`${RdiUrl.GetConfigTemplate}/${pipelineType}/${dbType}`);
       return response.data;
     } catch (error) {
       throw wrapRdiPipelineError(error);
     }
   }
 
-  async deploy(pipeline: RdiPipeline): Promise<void> {
-    let response;
+  async getJobTemplate(pipelineType: string): Promise<RdiTemplateResponseDto> {
     try {
-      response = await this.client.post(RdiUrl.Deploy, { ...pipeline });
+      const response = await this.client.get(`${RdiUrl.GetJobTemplate}/${pipelineType}`);
+      return response.data;
     } catch (error) {
-      throw wrapRdiPipelineError(error, error.response.data.message);
+      throw wrapRdiPipelineError(error);
     }
+  }
 
-    if (response.data?.status === RDI_DEPLOY_FAILED_STATUS) {
-      throw new RdiPipelineDeployFailedException(undefined, { error: response.data?.error });
+  async deploy(pipeline: RdiPipeline): Promise<void> {
+    try {
+      const response = await this.client.post(RdiUrl.Deploy, { ...pipeline });
+      const actionId = response.data.action_id;
+
+      return this.pollActionStatus(actionId);
+    } catch (error) {
+      throw wrapRdiPipelineError(error, error.response.data.message);
     }
   }
 
@@ -147,7 +156,7 @@ export class ApiRdiClient extends RdiClient {
 
   async getJobFunctions(): Promise<object> {
     try {
-      const response = await this.client.post(RdiUrl.JobFunctions);
+      const response = await this.client.get(RdiUrl.JobFunctions);
       return response.data;
     } catch (e) {
       throw wrapRdiPipelineError(e);
@@ -178,10 +187,10 @@ export class ApiRdiClient extends RdiClient {
     }
   }
 
-  private async pollActionStatus(actionId: string, abortSignal: AbortSignal): Promise<any> {
+  private async pollActionStatus(actionId: string, abortSignal?: AbortSignal): Promise<any> {
     const startTime = Date.now();
     while (true) {
-      if (abortSignal.aborted) {
+      if (abortSignal?.aborted) {
         throw new RdiPipelineInternalServerErrorException();
       }
       if (Date.now() - startTime > MAX_POLLING_TIME) {

redisinsight/api/src/modules/rdi/client/rdi.client.ts

@@ -4,7 +4,7 @@ import {
   RdiClientMetadata, RdiPipeline, RdiStatisticsResult,
 } from 'src/modules/rdi/models';
 import {
-  RdiDryRunJobDto, RdiDryRunJobResponseDto, RdiTestConnectionsResponseDto,
+  RdiDryRunJobDto, RdiDryRunJobResponseDto, RdiTemplateResponseDto, RdiTestConnectionsResponseDto,
 } from 'src/modules/rdi/dto';
 import { IDLE_TRESHOLD } from 'src/modules/rdi/constants';
 
@@ -28,8 +28,9 @@ export abstract class RdiClient {
 
   abstract getPipeline(): Promise<RdiPipeline>;
 
-  // TODO validate options and response
-  abstract getTemplate(options: object): Promise<unknown>;
+  abstract getConfigTemplate(pipelineType: string, dbType: string): Promise<RdiTemplateResponseDto>;
+
+  abstract getJobTemplate(pipelineType: string): Promise<RdiTemplateResponseDto>;
 
   abstract getStrategies(): Promise<object>;
 

redisinsight/api/src/modules/rdi/constants/index.ts

@@ -2,7 +2,8 @@ export enum RdiUrl {
   GetSchema = 'api/v1/schemas',
   GetPipeline = 'api/v1/pipelines',
   GetStrategies = 'api/v1/pipelines/strategies',
-  GetTemplate = 'deployments/templates',
+  GetConfigTemplate = 'api/v1/pipelines/config/templates',
+  GetJobTemplate = 'api/v1/pipelines/jobs/templates',
   DryRunJob = 'api/v1/pipelines/jobs/dry-run',
   JobFunctions = '/api/v1/pipelines/jobs/functions',
   Deploy = 'api/v1/pipelines',

redisinsight/api/src/modules/rdi/dto/index.ts

@@ -3,3 +3,4 @@ export * from './update.rdi.dto';
 export * from './rdi.dry-run.job.dto';
 export * from './rdi.dry-run.job.response.dto';
 export * from './rdi-test-connections.response.dto';
+export * from './rdi-template.response.dto';

redisinsight/api/src/modules/rdi/dto/rdi-template.response.dto.ts

@@ -0,0 +1,11 @@
+import { ApiProperty } from '@nestjs/swagger';
+import { Expose } from 'class-transformer';
+
+export class RdiTemplateResponseDto {
+  @ApiProperty({
+    description: 'Template for rdi file',
+    type: String,
+  })
+  @Expose()
+  template: string;
+}

redisinsight/api/src/modules/rdi/rdi-pipeline.controller.ts

@@ -1,15 +1,15 @@
 import {
   Body,
   ClassSerializerInterceptor, Controller, Get, Post, UseInterceptors, UsePipes, ValidationPipe,
-  Query, Req,
+  Req, Param,
 } from '@nestjs/common';
 import { Rdi, RdiPipeline, RdiClientMetadata } from 'src/modules/rdi/models';
 import { ApiTags } from '@nestjs/swagger';
 import { Request } from 'express';
 import { ApiEndpoint } from 'src/decorators/api-endpoint.decorator';
 import { RdiPipelineService } from 'src/modules/rdi/rdi-pipeline.service';
 import { RequestRdiClientMetadata } from 'src/modules/rdi/decorators';
-import { RdiDryRunJobDto, RdiTestConnectionsResponseDto } from 'src/modules/rdi/dto';
+import { RdiDryRunJobDto, RdiTemplateResponseDto, RdiTestConnectionsResponseDto } from 'src/modules/rdi/dto';
 import { RdiDryRunJobResponseDto } from 'src/modules/rdi/dto/rdi.dry-run.job.response.dto';
 
 @ApiTags('RDI')
@@ -91,16 +91,29 @@ export class RdiPipelineController {
     return this.rdiPipelineService.getStrategies(rdiClientMetadata);
   }
 
-  @Get('/template')
+  @Get('/job/template/:pipelineType')
   @ApiEndpoint({
-    description: 'Get pipeline template for selected pipeline type',
-    responses: [{ status: 200, type: Rdi }],
+    description: 'Get job template for selected pipeline type',
+    responses: [{ status: 200, type: RdiTemplateResponseDto }],
   })
-  async getTemplate(
+  async getJobTemplate(
     @RequestRdiClientMetadata() rdiClientMetadata: RdiClientMetadata,
-      @Query() options: object,
-  ): Promise<unknown> {
-    return this.rdiPipelineService.getTemplate(rdiClientMetadata, options);
+      @Param('pipelineType') pipelineType: string,
+  ): Promise<RdiTemplateResponseDto> {
+    return this.rdiPipelineService.getJobTemplate(rdiClientMetadata, pipelineType);
+  }
+
+  @Get('/config/template/:pipelineType/:dbType')
+  @ApiEndpoint({
+    description: 'Get config template for selected pipeline and db types',
+    responses: [{ status: 200, type: RdiTemplateResponseDto }],
+  })
+  async getConfigTemplate(
+    @RequestRdiClientMetadata() rdiClientMetadata: RdiClientMetadata,
+      @Param('pipelineType') pipelineType: string,
+      @Param('dbType') dbType: string,
+  ): Promise<RdiTemplateResponseDto> {
+    return this.rdiPipelineService.getConfigTemplate(rdiClientMetadata, pipelineType, dbType);
   }
 
   @Get('/status')

redisinsight/api/src/modules/rdi/rdi-pipeline.service.ts

@@ -2,7 +2,7 @@ import { Injectable, Logger } from '@nestjs/common';
 import { Request } from 'express';
 import { RdiClientMetadata, RdiPipeline } from 'src/modules/rdi/models';
 import { RdiClientProvider } from 'src/modules/rdi/providers/rdi.client.provider';
-import { RdiDryRunJobDto, RdiTestConnectionsResponseDto } from 'src/modules/rdi/dto';
+import { RdiDryRunJobDto, RdiTemplateResponseDto, RdiTestConnectionsResponseDto } from 'src/modules/rdi/dto';
 import { RdiDryRunJobResponseDto } from 'src/modules/rdi/dto/rdi.dry-run.job.response.dto';
 import { RdiPipelineAnalytics } from 'src/modules/rdi/rdi-pipeline.analytics';
 import { wrapHttpError } from 'src/common/utils';
@@ -85,12 +85,20 @@ export class RdiPipelineService {
     return await client.getStrategies();
   }
 
-  async getTemplate(rdiClientMetadata: RdiClientMetadata, options: object): Promise<unknown> {
-    this.logger.log('Getting RDI pipeline template');
+  async getConfigTemplate(rdiClientMetadata: RdiClientMetadata, pipelineType: string, dbType: string): Promise<RdiTemplateResponseDto> {
+    this.logger.log('Getting RDI config template');
 
     const client = await this.rdiClientProvider.getOrCreate(rdiClientMetadata);
 
-    return await client.getTemplate(options);
+    return await client.getConfigTemplate(pipelineType, dbType);
+  }
+
+  async getJobTemplate(rdiClientMetadata: RdiClientMetadata, pipelineType: string): Promise<RdiTemplateResponseDto> {
+    this.logger.log('Getting RDI job template');
+
+    const client = await this.rdiClientProvider.getOrCreate(rdiClientMetadata);
+
+    return await client.getJobTemplate(pipelineType);
   }
 
   async getPipelineStatus(rdiClientMetadata: RdiClientMetadata): Promise<unknown> {