[CYB-208] OCSF parser config support (#85)

* OCSF support parser config support

* switched from field selection to a text field

* Java tests fix

* Karma tests fix

* parsers base path fix

* minor refactoring to address the Observable subscription issues

* Hidden objects with unknown types

Author: stas-panasiuk

flink-cyber/flink-cyber-api/src/main/java/com/cloudera/cyber/indexing/MappingColumnDto.java

@@ -1,5 +1,6 @@
 package com.cloudera.cyber.indexing;
 
+import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.fasterxml.jackson.annotation.JsonProperty;
 import lombok.AllArgsConstructor;
 import lombok.Data;
@@ -26,6 +27,7 @@ public class MappingColumnDto {
     @JsonProperty("is_map")
     private Boolean isMap;
 
+    @JsonIgnore
     public String getKafkaName() {
         final String properName = kafkaName == null ? name : kafkaName;
         if (getIsMap()) {
@@ -38,6 +40,7 @@ public String getKafkaName() {
         }
     }
 
+    @JsonIgnore
     public String getRawKafkaName(){
         return kafkaName;
     }

flink-cyber/flink-indexing/flink-indexing-hive/src/main/java/com/cloudera/cyber/indexing/hive/util/AvroSchemaUtil.java

@@ -14,9 +14,13 @@
 import org.apache.flink.types.Row;
 
 import java.time.Instant;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
 
-public class AvroSchemaUtil {
+public final class AvroSchemaUtil {
 
     //method that converts from flink Schema to avro Schema
     public static Schema convertToAvro(List<TableColumnDto> tableColumnList) {

flink-cyber/flink-indexing/flink-indexing-hive/src/main/java/com/cloudera/cyber/indexing/hive/util/FlinkSchemaUtil.java

@@ -1,17 +1,18 @@
 package com.cloudera.cyber.indexing.hive.util;
 
 import com.cloudera.cyber.indexing.TableColumnDto;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Optional;
-import java.util.stream.Collectors;
 import org.apache.flink.table.api.DataTypes;
 import org.apache.flink.table.api.Schema;
 import org.apache.flink.table.catalog.Column;
 import org.apache.flink.table.catalog.ResolvedSchema;
 import org.apache.flink.table.types.DataType;
 
-public class FlinkSchemaUtil {
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+import java.util.stream.Collectors;
+
+public final class FlinkSchemaUtil {
 
     public static Schema buildSchema(ResolvedSchema resolvedSchema) {
         return Schema.newBuilder()

flink-cyber/metron-parser-chain/parser-chains-config-service/frontend/parser-chains-client/.eslintrc.json

@@ -107,7 +107,10 @@
           },
           {
             "selector": "memberLike",
-            "format": ["camelCase"],
+            "format": [
+              "camelCase",
+              "snake_case"
+            ],
             "leadingUnderscore": "forbid"
           },
           {
@@ -133,6 +136,15 @@
             "type": "attribute"
           }
         ],
+        "@typescript-eslint/ban-types": [
+          "error",
+          {
+            "extendDefaults": true,
+            "types": {
+              "{}": false
+            }
+          }
+        ],
         "@angular-eslint/no-empty-lifecycle-method": "error"
       }
     },

flink-cyber/metron-parser-chain/parser-chains-config-service/frontend/parser-chains-client/src/app/chain-page/chain-page.component.html

@@ -43,6 +43,9 @@
           <app-indexing-form
                   (fieldSetUpdated)="updateAllFields($event)"></app-indexing-form>
         </nz-tab>
+        <nz-tab class="ocsf-tab" nzTitle="OCSF builder">
+          <app-ocsf-form></app-ocsf-form>
+        </nz-tab>
       </nz-tabset>
     </div>
     <div nz-col nzSpan="14">

flink-cyber/metron-parser-chain/parser-chains-config-service/frontend/parser-chains-client/src/app/chain-page/chain-page.models.ts

@@ -59,3 +59,17 @@ export interface ChainDetailsModel {
   name: string;
   parsers: ParserModel[];
 }
+
+export interface IndexTableMapping {
+  table_name: string;
+  ignore_fields?: string[]
+  column_mapping: IndexingColumnMapping[]
+}
+
+export interface IndexingColumnMapping {
+  name: string;
+  kafka_name?: string;
+  path?: string;
+  transformation?: string;
+  is_map?: boolean;
+}

flink-cyber/metron-parser-chain/parser-chains-config-service/frontend/parser-chains-client/src/app/chain-page/chain-page.module.ts

@@ -52,6 +52,12 @@ import {NzLayoutModule} from "ng-zorro-antd/layout";
 import {NzAutocompleteModule} from "ng-zorro-antd/auto-complete";
 import {NzCheckboxModule} from "ng-zorro-antd/checkbox";
 import {MonacoEditorModule} from "ngx-monaco-editor-v2";
+import {OcsfFormComponent} from './components/ocsf-form/ocsf-form.component';
+import {MatCardModule} from "@angular/material/card";
+import {MatProgressSpinnerModule} from "@angular/material/progress-spinner";
+import {NzDividerModule} from "ng-zorro-antd/divider";
+import {NzSwitchModule} from "ng-zorro-antd/switch";
+import {OcsfObjectFormComponent} from './components/ocsf-form/ocsf-object-form/ocsf-object-form.component';
 
 @NgModule({
   declarations: [
@@ -66,6 +72,8 @@ import {MonacoEditorModule} from "ngx-monaco-editor-v2";
     AutofocusDirective,
     MultiInputComponent,
     IndexingFormComponent,
+    OcsfFormComponent,
+    OcsfObjectFormComponent,
   ],
   entryComponents: [ ChainViewComponent ],
   imports: [
@@ -95,6 +103,10 @@ import {MonacoEditorModule} from "ngx-monaco-editor-v2";
     NzLayoutModule,
     NzAutocompleteModule,
     NzCheckboxModule,
+    MatCardModule,
+    MatProgressSpinnerModule,
+    NzDividerModule,
+    NzSwitchModule,
   ],
   providers: [
     NzMessageService,

flink-cyber/metron-parser-chain/parser-chains-config-service/frontend/parser-chains-client/src/app/chain-page/components/live-view/live-view.effects.spec.ts

@@ -31,6 +31,7 @@ import {LiveViewEffects} from './live-view.effects';
 import {SampleDataType} from './models/sample-data.model';
 import {LiveViewService} from './services/live-view.service';
 import {EntryParsingResultModel} from "./models/live-view.model";
+import {provideMockStore} from "@ngrx/store/testing";
 
 describe('live-view.effects', () => {
 
@@ -46,11 +47,24 @@ describe('live-view.effects', () => {
     }
   };
 
-  const testResult:EntryParsingResultModel[] =
-     [
+  const selectedPipeline = 'foo-pipeline';
+  const chainListPageInitialState = {
+    items: [],
+    createModalVisible: false,
+    deleteModalVisible: false,
+    deleteItem: null,
+    loading: false,
+    error: '',
+    pipelines: null,
+    pipelineRenameModalVisible: false,
+    selectedPipeline: selectedPipeline
+  };
+
+  const testResult: EntryParsingResultModel[] =
+    [
       {
         output: 'output result',
-        log: { type: '', message: 'log result', stackTrace: '' },
+        log: {type: '', message: 'log result', stackTrace: ''},
       }
     ];
 
@@ -63,12 +77,20 @@ describe('live-view.effects', () => {
     TestBed.configureTestingModule({
       providers: [
         LiveViewEffects,
-        { provide: LiveViewService, useValue: jasmine.createSpyObj('LiveViewService', {
+        {
+          provide: LiveViewService, useValue: jasmine.createSpyObj('LiveViewService', {
             execute: of({results: testResult})
-          }) },
-        { provide: NzMessageService, useValue: jasmine.createSpyObj('NzMessageService', ['create']) },
-
-        provideMockActions(() => actions$)],
+          })
+        },
+        {provide: NzMessageService, useValue: jasmine.createSpyObj('NzMessageService', ['create'])},
+
+        provideMockActions(() => actions$),
+        provideMockStore({
+          initialState: {
+            'chain-list-page': chainListPageInitialState
+          },
+          selectors: []
+        })],
     });
 
     liveViewEffects = TestBed.inject(LiveViewEffects) as jasmine.SpyObj<LiveViewEffects>;
@@ -80,16 +102,16 @@ describe('live-view.effects', () => {
     const testSubscriber = jasmine.createSpy('executionTriggeredSpy');
     liveViewEffects.execute$.subscribe(testSubscriber);
 
-    actions$.next(executionTriggered({ ...testPayload }));
+    actions$.next(executionTriggered({...testPayload}));
 
-    expect(fakeLiveViewService.execute).toHaveBeenCalledWith(testPayload.sampleData, testPayload.chainConfig);
+    expect(fakeLiveViewService.execute).toHaveBeenCalledWith(testPayload.sampleData, testPayload.chainConfig, selectedPipeline);
   });
 
   it('should dispatch liveViewRefreshedSuccessfully if liveViewService execute successfully', () => {
     const testSubscriber = jasmine.createSpy('executionTriggeredSpy');
     liveViewEffects.execute$.subscribe(testSubscriber);
 
-    actions$.next(executionTriggered({ ...testPayload }));
+    actions$.next(executionTriggered({...testPayload}));
 
     expect(testSubscriber).toHaveBeenCalledWith({
       liveViewResult: {
@@ -103,9 +125,9 @@ describe('live-view.effects', () => {
     const testSubscriber = jasmine.createSpy('executionTriggeredSpy');
     liveViewEffects.execute$.subscribe(testSubscriber);
 
-    fakeLiveViewService.execute.and.returnValue(throwError({ message: 'something went wrong' }));
+    fakeLiveViewService.execute.and.returnValue(throwError({message: 'something went wrong'}));
 
-    actions$.next(executionTriggered({ ...testPayload }));
+    actions$.next(executionTriggered({...testPayload}));
 
     expect(testSubscriber).toHaveBeenCalledWith({
       error: {
@@ -116,11 +138,11 @@ describe('live-view.effects', () => {
   });
 
   it('should show error message if liveViewService execution fail', () => {
-    fakeLiveViewService.execute.and.returnValue(throwError({ message: 'something went wrong' }));
+    fakeLiveViewService.execute.and.returnValue(throwError({message: 'something went wrong'}));
 
     liveViewEffects.execute$.subscribe();
 
-    actions$.next(executionTriggered({ ...testPayload }));
+    actions$.next(executionTriggered({...testPayload}));
 
     expect(fakeMessageService.create).toHaveBeenCalledWith('error', 'something went wrong');
   });
@@ -130,24 +152,27 @@ describe('live-view.effects', () => {
 
     liveViewEffects.persistingSampleData$.subscribe();
 
-    actions$.next(sampleDataInputChanged({ sampleData: { type: SampleDataType.MANUAL, source: 'testing persistance' } }));
+    actions$.next(sampleDataInputChanged({sampleData: {type: SampleDataType.MANUAL, source: 'testing persistance'}}));
 
     expect(localStorage.setItem).toHaveBeenCalledWith(
       LiveViewConsts.SAMPLE_DATA_STORAGE_KEY,
-      JSON.stringify({ type: SampleDataType.MANUAL, source: 'testing persistance' })
+      JSON.stringify({type: SampleDataType.MANUAL, source: 'testing persistance'})
     );
   });
 
   it('should restore sample data input from local storage on liveViewInitialized action', () => {
     const testSubscriber = jasmine.createSpy('sampleDataRestoredSpy');
     liveViewEffects.restoreSampleDataFromLocalStore.subscribe(testSubscriber);
 
-    spyOn(localStorage, 'getItem').and.returnValue(JSON.stringify({ type: SampleDataType.MANUAL, source: 'persisted state' }));
+    spyOn(localStorage, 'getItem').and.returnValue(JSON.stringify({
+      type: SampleDataType.MANUAL,
+      source: 'persisted state'
+    }));
 
     actions$.next(liveViewInitialized());
 
     expect(testSubscriber).toHaveBeenCalledWith({
-      sampleData: { type: SampleDataType.MANUAL, source: 'persisted state' },
+      sampleData: {type: SampleDataType.MANUAL, source: 'persisted state'},
       type: sampleDataRestored.type
     });
   });
@@ -157,7 +182,7 @@ describe('live-view.effects', () => {
 
     liveViewEffects.persistingOnOffToggle$.subscribe();
 
-    actions$.next(onOffToggleChanged({ value: true }));
+    actions$.next(onOffToggleChanged({value: true}));
 
     expect(localStorage.setItem).toHaveBeenCalledWith(
       LiveViewConsts.FEATURE_TOGGLE_STORAGE_KEY,
@@ -175,7 +200,7 @@ describe('live-view.effects', () => {
 
     expect(testSubscriber).toHaveBeenCalledWith({
       value: true,
-      type: onOffToggleRestored .type
+      type: onOffToggleRestored.type
     });
   });
 

flink-cyber/metron-parser-chain/parser-chains-config-service/frontend/parser-chains-client/src/app/chain-page/components/live-view/live-view.effects.ts

@@ -12,10 +12,10 @@
 
 import {Injectable} from '@angular/core';
 import {Actions, createEffect, ofType} from '@ngrx/effects';
-import {Action} from '@ngrx/store';
+import {Action, Store} from '@ngrx/store';
 import {NzMessageService} from 'ng-zorro-antd/message';
 import {Observable, of} from 'rxjs';
-import {catchError, map, switchMap, tap} from 'rxjs/operators';
+import {catchError, map, switchMap, tap, withLatestFrom} from 'rxjs/operators';
 
 import {
   executionTriggered,
@@ -30,6 +30,7 @@ import {
 } from './live-view.actions';
 import {LiveViewConsts} from './live-view.consts';
 import {LiveViewService} from './services/live-view.service';
+import {ChainListPageState, getSelectedPipeline} from "../../../chain-list-page/chain-list-page.reducers";
 
 @Injectable()
 export class LiveViewEffects {
@@ -38,8 +39,9 @@ export class LiveViewEffects {
     ofType(
       executionTriggered.type,
     ),
-    switchMap(({ sampleData, chainConfig }) => {
-      return this._liveViewService.execute(sampleData, chainConfig).pipe(
+    withLatestFrom(this._store$.select(getSelectedPipeline)),
+    switchMap(([{sampleData, chainConfig}, selectedPipeline]) => {
+      return this._liveViewService.execute(sampleData, chainConfig, selectedPipeline).pipe(
         map(liveViewResult => liveViewRefreshedSuccessfully({ liveViewResult })),
         catchError(( error: { message: string }) => {
           this._messageService.create('error', error.message);
@@ -89,6 +91,7 @@ export class LiveViewEffects {
 
   constructor(
     private _actions$: Actions<LiveViewActionsType>,
+    private _store$: Store<ChainListPageState>,
     private _liveViewService: LiveViewService,
     private _messageService: NzMessageService,
   ) {}

flink-cyber/metron-parser-chain/parser-chains-config-service/frontend/parser-chains-client/src/app/chain-page/components/live-view/services/live-view.service.spec.ts

@@ -10,13 +10,13 @@
  * limitations governing your use of the file.
  */
 
-import { HttpClient } from '@angular/common/http';
-import { HttpClientTestingModule } from '@angular/common/http/testing';
+import {HttpClient, HttpParams} from '@angular/common/http';
+import {HttpClientTestingModule} from '@angular/common/http/testing';
 import {TestBed} from '@angular/core/testing';
 
-import { SampleDataType } from '../models/sample-data.model';
+import {SampleDataType} from '../models/sample-data.model';
 
-import { LiveViewService } from './live-view.service';
+import {LiveViewService} from './live-view.service';
 
 describe('LiveViewService', () => {
   let service: LiveViewService;
@@ -40,16 +40,17 @@ describe('LiveViewService', () => {
     spyOn(http, 'post');
 
     service.execute(
-      { type: SampleDataType.MANUAL, source: 'test sample input' },
-      { id: '456', name: 'gdf', parsers: [] }
+      {type: SampleDataType.MANUAL, source: 'test sample input'},
+      {id: '456', name: 'gdf', parsers: []}
     );
 
     expect(http.post).toHaveBeenCalledWith(
       LiveViewService.BASE_URL,
       {
-        sampleData: { type: SampleDataType.MANUAL, source: ['test sample input'] },
-        chainConfig: { id: '456', name: 'gdf', parsers: [] }
-      }
+        sampleData: {type: SampleDataType.MANUAL, source: ['test sample input']},
+        chainConfig: {id: '456', name: 'gdf', parsers: []}
+      },
+      {params: new HttpParams()}
     );
   });
 });