Merge pull request #148 from Patowhiz/pwa

Fixes bugs and makes data checking improvements

Author: Patowhiz

back-end/api/package.json

@@ -1,6 +1,6 @@
 {
   "name": "api",
-  "version": "preview-2.0.0",
+  "version": "preview-2.0.1",
   "description": "Climsoft API",
   "author": "Climsoft Foundation",
   "private": true,

back-end/api/src/main.ts

@@ -1,10 +1,11 @@
 import { NestFactory } from '@nestjs/core';
 import { AppModule } from './app.module';
-import { ValidationPipe } from '@nestjs/common'; 
+import { ValidationPipe } from '@nestjs/common';
 import { AppConfig } from './app.config';
 import * as session from 'express-session';
 import * as pgSession from 'connect-pg-simple';
 import { Pool } from 'pg';
+import * as express from 'express';
 
 async function bootstrap() {
   const app = await NestFactory.create(AppModule);
@@ -23,11 +24,11 @@ async function bootstrap() {
     // origin:  process.env.WEB_APP_BASE_URLs ? process.env.WEB_APP_BASE_URLs : 'http://localhost:4200' , 
     origin: (origin, callback) => {
       //console.log(`Client Origin - ${origin}`); 
-      
+
       // TODO. In future implement CORs security feature to enable users to determine origin setting based on their security requirements.
       callback(null, true); // Allow the request
-  
-   
+
+
       // TODO Code below is meant to enfors allowed origins
       // Allow requests with no `Origin` (e.g., from desktop and mobile apps)
       // Only allows requests from trusted web app origins. This is needed because web browsers require it
@@ -77,21 +78,11 @@ async function bootstrap() {
     }),
   );
 
-  // Left here for reference. This uses the default MemoryStore that is not designed for production
-  // app.use(
-  //   session({
-  //     name: 'ssid',
-  //     secret: AppConfig.dbCredentials.password,
-  //     resave: false,
-  //     saveUninitialized: false,
-  //     cookie: {
-  //       maxAge: 1000 * 60 * 60 * 24, // set to 24 hours
-  //       sameSite: false,
-  //       secure: false //TODO set to true only when using HTTPS
-  //     },
-  //   }),
-  // );
+  // Increase the allowed payload request from the default 100kb to 1MB
+  app.use(express.json({ limit: '1mb' }));
+  app.use(express.urlencoded({ limit: '1mb', extended: true }));
 
+  // Set the port to listen for connections
   await app.listen(3000);
 }
 

back-end/api/src/metadata/source-templates/services/source-templates.service.ts

@@ -93,28 +93,32 @@ export class SourceTemplatesService {
 
         await this.sourceRepo.save(entity);
 
-        this.eventEmitter.emit('source.created', { id: entity.id, dto });
+        const viewDto: ViewSourceDto = this.createViewDto(entity);
 
-        return this.createViewDto(entity);
+        this.eventEmitter.emit('source.created', { id: entity.id, viewDto });
+
+        return viewDto;
 
     }
 
-    public async update(id: number, dto: CreateUpdateSourceDto, userId: number) {
-        const source = await this.findEntity(id);
-        source.name = dto.name;
-        source.description = dto.description;
-        source.sourceType = dto.sourceType;
-        source.utcOffset = dto.utcOffset;
-        source.allowMissingValue = dto.allowMissingValue;
-        source.sampleImage = dto.sampleImage;
-        source.parameters = dto.parameters;
-        source.entryUserId = userId;
+    public async update(id: number, dto: CreateUpdateSourceDto, userId: number): Promise<ViewSourceDto> {
+        const entity = await this.findEntity(id);
+        entity.name = dto.name;
+        entity.description = dto.description;
+        entity.sourceType = dto.sourceType;
+        entity.utcOffset = dto.utcOffset;
+        entity.allowMissingValue = dto.allowMissingValue;
+        entity.sampleImage = dto.sampleImage;
+        entity.parameters = dto.parameters;
+        entity.entryUserId = userId;
+
+        await this.sourceRepo.save(entity);
 
-        await this.sourceRepo.save(source);
+        const viewDto: ViewSourceDto = this.createViewDto(entity);
 
-        this.eventEmitter.emit('source.updated', { id, dto });
+        this.eventEmitter.emit('source.updated', { id, viewDto });
 
-        return source;
+        return viewDto;
     }
 
     public async delete(id: number): Promise<number> {
@@ -132,7 +136,7 @@ export class SourceTemplatesService {
         return true;
     }
 
-    private async createViewDto(entity: SourceTemplateEntity): Promise<ViewSourceDto> {
+    private createViewDto(entity: SourceTemplateEntity): ViewSourceDto {
         const dto: ViewSourceDto = {
             id: entity.id,
             name: entity.name,

back-end/api/src/migrations/migrations.service.ts

@@ -77,9 +77,7 @@ export class MigrationsService {
     await this.seedTriggers();
     await this.seedFirstUser();
     await this.seedMetadata();
-
-    // Default general settings
-    await this.generalSettingsService.bulkPut(GeneralSettingsDefaults.GENERAL_SETTINGS, 1);
+    await this.seedGeneralSettings();
   }
 
   private async seedTriggers() {
@@ -91,7 +89,7 @@ export class MigrationsService {
   private async seedFirstUser() {
     const count = await this.userService.count();
     if (count === 0) {
-      const newUser = await this.userService.createUser(
+      const newUser = await this.userService.create(
         {
           name: "admin",
           email: "admin@climsoft.org",
@@ -110,7 +108,7 @@ export class MigrationsService {
     }
   }
 
-  public async seedMetadata() {
+  private async seedMetadata() {
     let count: number;
     // Elements metadata
     count = await this.elementSubdomainsService.count();
@@ -140,6 +138,13 @@ export class MigrationsService {
 
   }
 
-
+  private async seedGeneralSettings() {
+    // Default general settings
+    const count: number = await this.generalSettingsService.count();
+    if (count === 0) {
+      await this.generalSettingsService.bulkPut(GeneralSettingsDefaults.GENERAL_SETTINGS, 1);
+      this.logger.log('general settings added');
+    }
+  }
 
 }

back-end/api/src/observation/controllers/observations.controller.ts

@@ -8,7 +8,6 @@ import { AuthorisedStationsPipe } from 'src/user/pipes/authorised-stations.pipe'
 import { Request } from 'express';
 import { AuthUtil } from 'src/user/services/auth.util';
 import { EntryFormObservationQueryDto } from '../dtos/entry-form-observation-query.dto';
-import { ViewObservationLogQueryDto } from '../dtos/view-observation-log-query.dto';
 import { DeleteObservationDto } from '../dtos/delete-observation.dto';
 import { Admin } from 'src/user/decorators/admin.decorator';
 import { ExportObservationsService } from '../services/export-observations.service';
@@ -50,23 +49,6 @@ export class ObservationsController {
     return this.observationsService.findFormData(createObsevationQuery);
   }
 
-  // TODO. deprecate this handle
-  //@Get('correction-data')
-  //getCorrectionData(@Query(AuthorisedStationsPipe) viewObsevationQuery: ViewObservationQueryDTO) {
-    //return this.observationsService.findCorrectionData(viewObsevationQuery);
-    //return this.observationsService.findProcessed(viewObsevationQuery);
-  //}
-
-  //@Get('count-correction-data')
-  //countCorrectionData(@Query(AuthorisedStationsPipe) viewObsevationQuery: ViewObservationQueryDTO) {
-  //  return this.observationsService.count(viewObsevationQuery);
-  //}
-
-  @Get('log')
-  getObservationLog(@Query(AuthorisedStationsPipe) viewObsevationQuery: ViewObservationLogQueryDto) {
-    return this.observationsService.findObservationLog(viewObsevationQuery);
-  }
-
   @Get('stations-observation-status')
   getStationsObservationStatus(@Query(AuthorisedStationsPipe) stationStatusQuery: StationStatusQueryDto) { // TODO. Create dto query to make the necessary filter
     return this.observationsService.findStationsStatus(stationStatusQuery);

back-end/api/src/observation/dtos/view-observation-log.dto.ts

@@ -7,6 +7,7 @@ export class ViewObservationLogDto {
   qcStatus: QCStatusEnum;
   comment: string | null;
   deleted: boolean;
+  entryUserName: string;
   entryUserEmail: string;
   entryDateTime: string;
 }

back-end/api/src/observation/services/data-entry-check.service.ts

@@ -33,8 +33,7 @@ export class DataEntryCheckService {
 
     @OnEvent('source.created')
     handleSourceCreated(payload: { id: number; dto: any }) {
-        console.log(`Source created: ID ${payload.id}`);
-        // maybe invalidate cache, trigger sync, etc.
+        console.log(`Source created: ID ${payload.id}`); 
         this.resetFormParameters();
     }
 
@@ -74,7 +73,6 @@ export class DataEntryCheckService {
             } else {
                 throw new Error('Developer error: Source type not recognised')
             }
-
         }
     }
 
@@ -95,32 +93,49 @@ export class DataEntryCheckService {
 
             const source = this.sourceParameters.get(dto.sourceId);
 
-            if (!source) throw new BadRequestException('Source template not found');
+            if (!source) {
+                this.logger.error(`Source template not found. Dto:  ${dto} | User: ${user}`)
+                throw new BadRequestException('Source template not found');
+            }
 
             if (source.sourceType === SourceTypeEnum.FORM) {
                 const formTemplate: FormParams = source.settings as FormParams;
                 // check element
-                if (!formTemplate.form.elementIds.includes(dto.elementId)) throw new BadRequestException('Element not allowed');
+                if (!formTemplate.form.elementIds.includes(dto.elementId)) {
+                    this.logger.error(`Element not allowed. Dto:  ${dto} | User: ${user}`);
+                    throw new BadRequestException('Element not allowed');
+                }
 
                 // Check if hour is allowed for the form
-                if (!formTemplate.utcAdjustedHours.includes(parseInt(dto.datetime.substring(11, 13), 10))) throw new BadRequestException('Hour not allowed');
+                if (!formTemplate.utcAdjustedHours.includes(parseInt(dto.datetime.substring(11, 13), 10))) {
+                    this.logger.error(`Hour not allowed. Dto:  ${dto} | User: ${user}`);
+                    throw new BadRequestException('Hour not allowed');
+                }
 
                 // Check for interval is allowed for the form
                 if (!formTemplate.form.allowIntervalEditing) {
-                    if (formTemplate.form.interval !== dto.interval) throw new BadRequestException('Interval not allowed');
+                    if (formTemplate.form.interval !== dto.interval) {
+                        this.logger.error(`Interval not allowed. Dto:  ${dto} | User: ${user}`);
+                        throw new BadRequestException('Interval not allowed');
+                    }
                 }
 
             } else if (source.sourceType === SourceTypeEnum.IMPORT) {
-                // TODO. Use source params to validate the inut. Use full when doing data correction
+                // TODO. Use source params to validate the input. Use full when doing data correction
+                // TODO. Should import validation use DuckDB database engine?
             }
 
             // Check for future dates           
             if (new Date(dto.datetime) > todayDate) {
                 // TODO. Follow up on when invalid dates are being bypassed at the front end. 
-                this.logger.log(`obs datetime: ${dto.datetime} todayDate: ${todayDate.toISOString()}`);
+                this.logger.error(`Future dates not allowed. Dto ${dto} | User: ${user} | TodayDate: ${todayDate.toISOString()}`);
                 throw new BadRequestException('Future dates not allowed');
             }
 
+            if (dto.value === null && dto.flag === null) {
+                this.logger.error(`Both value and flag are missing. Dto:  ${dto} | User: ${user}`);
+                throw new BadRequestException('Both value and flag are missing');
+            }
         }
 
         this.logger.log(`observations checks took: ${new Date().getTime() - startTime} milliseconds`);