IOT-1610: Added new metadata field for ODDK datatargets (#286)

* Wip

* More work

* Last fixes

* Package update

* went through endpoints to verify no extra data should be shown

* npm i

* addd minimal on org

* get one with security and removed minimal from org controller

* Added new metadata field for ODDK datatargets

* Fix chirpstack gateway tags format to handle non-string JSON values like false

* Also fix gateway edit + device profile

* Bugfix/#206 gateway metadata error (#287)

* Fix chirpstack gateway tags format to handle non-string JSON values like false

* Also fix gateway edit + device profile

* Removed some unused imports

---------

Co-authored-by: August Andersen <aha@iterator-it.dk>

Author: fcv-iteratorIt

src/entities/dto/create-open-data-dk-dataset.dto.ts

@@ -1,5 +1,5 @@
 import { ApiProperty, ApiPropertyOptional } from "@nestjs/swagger";
-import { IsEmail, IsNotEmpty, IsOptional, IsString, IsUrl } from "class-validator";
+import { IsBoolean, IsEmail, IsNotEmpty, IsOptional, IsString, IsUrl } from "class-validator";
 
 export class CreateOpenDataDkDatasetDto {
   @ApiProperty({ required: true })
@@ -17,6 +17,11 @@ export class CreateOpenDataDkDatasetDto {
   @IsString({ each: true, always: true })
   keywords?: string[];
 
+  @ApiProperty({ required: false })
+  @IsOptional()
+  @IsString()
+  keywordTags: string;
+
   @ApiProperty({ required: true })
   @IsString()
   @IsUrl({ protocols: ["http", "https"] })
@@ -36,4 +41,18 @@ export class CreateOpenDataDkDatasetDto {
   @IsString()
   @IsNotEmpty()
   resourceTitle: string;
+
+  @ApiProperty({ required: false })
+  @IsOptional()
+  @IsString()
+  updateFrequency: string = "UNKNOWN";
+
+  @ApiProperty({ required: false })
+  @IsOptional()
+  @IsUrl()
+  documentationUrl: string;
+
+  @ApiProperty({ required: true })
+  @IsBoolean()
+  dataDirectory: boolean;
 }

src/entities/dto/open-data-dk-dcat.dto.ts

@@ -37,6 +37,9 @@ export class Dataset {
   distribution: Distribution[];
   spatial: string;
   theme: string[];
+  documentation: string;
+  frequency: string | undefined;
+  dataDirectory: boolean;
 }
 
 export class DCATRootObject {

src/entities/open-data-dk-dataset.entity.ts

@@ -18,6 +18,9 @@ export class OpenDataDkDataset extends DbBaseEntity {
   @Column("text", { array: true, nullable: true })
   keywords?: string[];
 
+  @Column({ nullable: true })
+  keywordTags?: string;
+
   @Column()
   license: string;
 
@@ -29,4 +32,13 @@ export class OpenDataDkDataset extends DbBaseEntity {
 
   @Column({ nullable: false, default: "" })
   resourceTitle: string;
+
+  @Column({ nullable: true })
+  updateFrequency?: string;
+
+  @Column({ nullable: true })
+  documentationUrl?: string;
+
+  @Column({ nullable: false, default: false })
+  dataDirectory: boolean;
 }

src/migration/1762524771197-added-new-fields-to-oddk-datatarget.ts

@@ -0,0 +1,20 @@
+import { MigrationInterface, QueryRunner } from "typeorm";
+
+export class AddedNewFieldsToOddkDatatarget1762524771197 implements MigrationInterface {
+    name = 'AddedNewFieldsToOddkDatatarget1762524771197'
+
+    public async up(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`ALTER TABLE "open_data_dk_dataset" ADD "keywordTags" character varying`);
+        await queryRunner.query(`ALTER TABLE "open_data_dk_dataset" ADD "updateFrequency" character varying`);
+        await queryRunner.query(`ALTER TABLE "open_data_dk_dataset" ADD "documentationUrl" character varying`);
+        await queryRunner.query(`ALTER TABLE "open_data_dk_dataset" ADD "dataDirectory" boolean NOT NULL DEFAULT false`);
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`ALTER TABLE "open_data_dk_dataset" DROP COLUMN "dataDirectory"`);
+        await queryRunner.query(`ALTER TABLE "open_data_dk_dataset" DROP COLUMN "documentationUrl"`);
+        await queryRunner.query(`ALTER TABLE "open_data_dk_dataset" DROP COLUMN "updateFrequency"`);
+        await queryRunner.query(`ALTER TABLE "open_data_dk_dataset" DROP COLUMN "keywordTags"`);
+    }
+
+}

src/services/chirpstack/chirpstack-gateway.service.ts

@@ -1,7 +1,7 @@
 import {
-  Gateway as ChirpstackGateway,
   CreateGatewayRequest,
   DeleteGatewayRequest,
+  Gateway as ChirpstackGateway,
   GetGatewayMetricsRequest,
   GetGatewayMetricsResponse,
   GetGatewayRequest,
@@ -53,6 +53,12 @@ import { Repository } from "typeorm";
 
 @Injectable()
 export class ChirpstackGatewayService extends GenericChirpstackConfigurationService {
+  GATEWAY_STATS_INTERVAL_IN_DAYS = 29;
+  GATEWAY_LAST_ACTIVE_SINCE_IN_MINUTES = 3;
+  private readonly logger = new Logger(ChirpstackGatewayService.name, {
+    timestamp: true,
+  });
+
   constructor(
     @InjectRepository(DbGateway)
     private gatewayRepository: Repository<DbGateway>,
@@ -62,11 +68,6 @@ export class ChirpstackGatewayService extends GenericChirpstackConfigurationServ
   ) {
     super();
   }
-  GATEWAY_STATS_INTERVAL_IN_DAYS = 29;
-  GATEWAY_LAST_ACTIVE_SINCE_IN_MINUTES = 3;
-  private readonly logger = new Logger(ChirpstackGatewayService.name, {
-    timestamp: true,
-  });
 
   async createNewGateway(dto: CreateGatewayDto, userId: number): Promise<ChirpstackResponseStatus> {
     dto.gateway = await this.updateDtoContents(dto.gateway);
@@ -87,7 +88,7 @@ export class ChirpstackGatewayService extends GenericChirpstackConfigurationServ
 
     const gatewayChirpstack = await this.mapToChirpstackGateway(dto, chirpstackLocation);
     Object.entries(dto.gateway.tags).forEach(([key, value]) => {
-      gatewayChirpstack.getTagsMap().set(key, value);
+      gatewayChirpstack.getTagsMap().set(key, value.toString());
     });
 
     req.setGateway(gatewayChirpstack);
@@ -302,54 +303,6 @@ export class ChirpstackGatewayService extends GenericChirpstackConfigurationServ
     }
   }
 
-  //TODO: This could be moved to a helper function in the future, since it has a lot of similarities with metrics from chirpstack devices.
-  private mapPackets(metrics: GetGatewayMetricsResponse) {
-    const gatewayResponseDto: GatewayStatsElementDto[] = [];
-    const packetCounts: { [timestamp: string]: { rx: number; tx: number } } = {};
-
-    const rxTimestamps = metrics.getRxPackets().getTimestampsList();
-    const rxPackets = metrics
-      .getRxPackets()
-      .getDatasetsList()
-      .find(e => e.getLabel() === "rx_count")
-      .getDataList();
-
-    this.processPackets(rxTimestamps, rxPackets, "rx", packetCounts);
-
-    const txTimestamps = metrics.getTxPackets().getTimestampsList();
-    const txPackets = metrics
-      .getTxPackets()
-      .getDatasetsList()
-      .find(e => e.getLabel() === "tx_count")
-      .getDataList();
-
-    this.processPackets(txTimestamps, txPackets, "tx", packetCounts);
-
-    Object.keys(packetCounts).forEach(timestamp => {
-      const packetCount = packetCounts[timestamp];
-      const dto: GatewayStatsElementDto = {
-        timestamp,
-        rxPacketsReceived: packetCount.rx,
-        txPacketsEmitted: packetCount.tx,
-      };
-      gatewayResponseDto.push(dto);
-    });
-    return gatewayResponseDto;
-  }
-
-  private processPackets(
-    timestamps: Array<Timestamp>,
-    packets: number[],
-    key: string,
-    packetCounts: { [timestamp: string]: { rx: number; tx: number } }
-  ) {
-    timestamps.forEach((timestamp, index) => {
-      const isoTimestamp = timestamp.toDate().toISOString();
-      packetCounts[isoTimestamp] = packetCounts[isoTimestamp] || { rx: 0, tx: 0 };
-      (packetCounts[isoTimestamp] as any)[key] = packets[index];
-    });
-  }
-
   async modifyGateway(
     gatewayId: string,
     dto: UpdateGatewayDto,
@@ -370,7 +323,7 @@ export class ChirpstackGatewayService extends GenericChirpstackConfigurationServ
     const gatewayCs = await this.mapToChirpstackGateway(dto, location, gatewayId);
 
     Object.entries(dto.gateway.tags).forEach(([key, value]) => {
-      gatewayCs.getTagsMap().set(key, value);
+      gatewayCs.getTagsMap().set(key, value.toString());
     });
 
     request.setGateway(gatewayCs);
@@ -447,20 +400,6 @@ export class ChirpstackGatewayService extends GenericChirpstackConfigurationServ
     }
   }
 
-  private async updateDtoContents(
-    contentsDto: GatewayContentsDto | UpdateGatewayContentsDto
-  ): Promise<GatewayContentsDto | UpdateGatewayContentsDto> {
-    if (contentsDto?.tagsString) {
-      contentsDto.tags = JSON.parse(contentsDto.tagsString);
-    } else {
-      contentsDto.tags = {};
-    }
-
-    contentsDto.id = contentsDto.gatewayId;
-
-    return contentsDto;
-  }
-
   public mapContentsDtoToGateway(dto: GatewayContentsDto) {
     const gateway = new DbGateway();
     gateway.name = dto.name;
@@ -536,24 +475,6 @@ export class ChirpstackGatewayService extends GenericChirpstackConfigurationServ
 
     return gateway;
   }
-  private mapGatewayToResponseDto(gateway: DbGateway, forMap = false): GatewayResponseDto {
-    const responseDto = gateway as unknown as GatewayResponseDto;
-    responseDto.organizationId = gateway.organization.id;
-    responseDto.organizationName = gateway.organization.name;
-
-    const commonLocation = new CommonLocationDto();
-    commonLocation.latitude = gateway.location.coordinates[1];
-    commonLocation.longitude = gateway.location.coordinates[0];
-
-    if (!forMap) {
-      commonLocation.altitude = gateway.altitude;
-      responseDto.tags = JSON.parse(gateway.tags);
-    }
-
-    responseDto.location = commonLocation;
-
-    return responseDto;
-  }
 
   async getAllGatewaysFromChirpstack(): Promise<ListAllChirpstackGatewaysResponseDto> {
     const limit = 1000;
@@ -587,24 +508,6 @@ export class ChirpstackGatewayService extends GenericChirpstackConfigurationServ
     return responseList;
   }
 
-  private getSortingForGateways(query: ListAllEntitiesDto) {
-    let orderBy = "gateway.id";
-
-    if (!query.orderOn) {
-      return orderBy;
-    }
-
-    if (query.orderOn === "organizationName") {
-      orderBy = "organization.name";
-    } else if (query.orderOn === "status") {
-      orderBy = "gateway.lastSeenAt";
-    } else {
-      orderBy = `gateway.${query.orderOn}`;
-    }
-
-    return orderBy;
-  }
-
   validatePackageAlarmInput(dto: UpdateGatewayDto) {
     if (dto.gateway.minimumPackages > dto.gateway.maximumPackages) {
       throw new BadRequestException({
@@ -628,6 +531,105 @@ export class ChirpstackGatewayService extends GenericChirpstackConfigurationServ
     }
   }
 
+  //TODO: This could be moved to a helper function in the future, since it has a lot of similarities with metrics from chirpstack devices.
+  private mapPackets(metrics: GetGatewayMetricsResponse) {
+    const gatewayResponseDto: GatewayStatsElementDto[] = [];
+    const packetCounts: { [timestamp: string]: { rx: number; tx: number } } = {};
+
+    const rxTimestamps = metrics.getRxPackets().getTimestampsList();
+    const rxPackets = metrics
+      .getRxPackets()
+      .getDatasetsList()
+      .find(e => e.getLabel() === "rx_count")
+      .getDataList();
+
+    this.processPackets(rxTimestamps, rxPackets, "rx", packetCounts);
+
+    const txTimestamps = metrics.getTxPackets().getTimestampsList();
+    const txPackets = metrics
+      .getTxPackets()
+      .getDatasetsList()
+      .find(e => e.getLabel() === "tx_count")
+      .getDataList();
+
+    this.processPackets(txTimestamps, txPackets, "tx", packetCounts);
+
+    Object.keys(packetCounts).forEach(timestamp => {
+      const packetCount = packetCounts[timestamp];
+      const dto: GatewayStatsElementDto = {
+        timestamp,
+        rxPacketsReceived: packetCount.rx,
+        txPacketsEmitted: packetCount.tx,
+      };
+      gatewayResponseDto.push(dto);
+    });
+    return gatewayResponseDto;
+  }
+
+  private processPackets(
+    timestamps: Array<Timestamp>,
+    packets: number[],
+    key: string,
+    packetCounts: { [timestamp: string]: { rx: number; tx: number } }
+  ) {
+    timestamps.forEach((timestamp, index) => {
+      const isoTimestamp = timestamp.toDate().toISOString();
+      packetCounts[isoTimestamp] = packetCounts[isoTimestamp] || { rx: 0, tx: 0 };
+      (packetCounts[isoTimestamp] as any)[key] = packets[index];
+    });
+  }
+
+  private async updateDtoContents(
+    contentsDto: GatewayContentsDto | UpdateGatewayContentsDto
+  ): Promise<GatewayContentsDto | UpdateGatewayContentsDto> {
+    if (contentsDto?.tagsString) {
+      contentsDto.tags = JSON.parse(contentsDto.tagsString);
+    } else {
+      contentsDto.tags = {};
+    }
+
+    contentsDto.id = contentsDto.gatewayId;
+
+    return contentsDto;
+  }
+
+  private mapGatewayToResponseDto(gateway: DbGateway, forMap = false): GatewayResponseDto {
+    const responseDto = gateway as unknown as GatewayResponseDto;
+    responseDto.organizationId = gateway.organization.id;
+    responseDto.organizationName = gateway.organization.name;
+
+    const commonLocation = new CommonLocationDto();
+    commonLocation.latitude = gateway.location.coordinates[1];
+    commonLocation.longitude = gateway.location.coordinates[0];
+
+    if (!forMap) {
+      commonLocation.altitude = gateway.altitude;
+      responseDto.tags = JSON.parse(gateway.tags);
+    }
+
+    responseDto.location = commonLocation;
+
+    return responseDto;
+  }
+
+  private getSortingForGateways(query: ListAllEntitiesDto) {
+    let orderBy = "gateway.id";
+
+    if (!query.orderOn) {
+      return orderBy;
+    }
+
+    if (query.orderOn === "organizationName") {
+      orderBy = "organization.name";
+    } else if (query.orderOn === "status") {
+      orderBy = "gateway.lastSeenAt";
+    } else {
+      orderBy = `gateway.${query.orderOn}`;
+    }
+
+    return orderBy;
+  }
+
   private async checkForNotificationUnusualPackagesAlarms(gateway: GatewayResponseDto) {
     if (!gateway.lastSeenAt) {
       return;

src/services/chirpstack/device-profile.service.ts

@@ -52,7 +52,7 @@ export class DeviceProfileService extends GenericChirpstackConfigurationService
     const deviceProfile = this.mapToChirpstackDto(dto, true);
 
     Object.entries(dto.deviceProfile.tags).forEach(([key, value]) => {
-      deviceProfile.getTagsMap().set(key, value);
+      deviceProfile.getTagsMap().set(key, value.toString());
     });
 
     req.setDeviceProfile(deviceProfile);