Roadmap features: Mqtt broker/subscriber support (#213)

* Added missing packages to package lock

* Create .licrc for licensebat scan

* Deleting licensebat file as it is obsolete

* IOT-1416: Minor changes for supporting the front-end-changes for having OpenDataDK datatargets more separate from HTTP-Push..

* Finished implementing new flow for sending ODDK-registration-mails directly from OS2IoT, instead of having to go through users mail-client..

* Shell framework for new device type, currently not working

* Renamed the new entity for OpenDataDK datatargets, to match naming-convention of other ODDK-files..

* Now possible to create broker, not fully populated

* Added new migration with correct enum for authenticationType

* Most broker options done. Cert version not implemented

* Fix missed enum file rename

* MQTT publisher and subscriber almost fully implemented

* Added encryption of iot-device secret fields

* added openssl to dockerfile

* csv export half baked

* More unfinished work on csv generator

* Csv generator done

* Small fixes

* updated vm2 version

* Pr fixes

* Reworked marking device as invalid for mqtt-clients

* made fallback on ENCRYPTION_SYMMETRIC_KEY and changed fallback on CA_KEY_PASSWORD.

* Added error checking for required fields on mqtt units

* Added CA certificate to brokers using password auth on fetch

* Changed error message on mqtt certificate rows

* Moved hashed version of mqtt password to own column

* Added new columns to csv export

* Removed redundant todo

* Add id back to csv export

* Removed some unused code

* Recreate mqtt subscriber client on edit

* Validate create many input

* Sæt min length on mqtt username and password (Previously an empty string was accepted

* Deleted wrongly placed migration

* fixed port error and made sure that mqtt will be removed when unable to connect.

* Renamed MqttBroker -> MqttInternalBroker and MqttSubscriber -> MqttExternalBroker

* Added migrations for renaming

---------

Co-authored-by: Nikolaj Gustafsson <[email protected]>
Co-authored-by: OS2 primary account <[email protected]>
Co-authored-by: Chris Johansen <[email protected]>
Co-authored-by: August Andersen <[email protected]>

Author: 5 people

.gitignore

@@ -1,7 +1,8 @@
 # compiled output
 /dist
 /node_modules
-
+ca.crt
+ca.key
 # Logs
 logs
 *.log

Dockerfile

@@ -5,6 +5,8 @@ ENV NODE_ENV build
 
 RUN npm install -g nest eslint jest
 
+RUN apk add openssl
+
 USER node
 
 # ENV NPM_CONFIG_PREFIX=/home/node/.npm-global

package-lock.json

@@ -44,6 +44,7 @@
         "@types/geojson": "^7946.0.7",
         "@types/kafkajs": "^1.9.0",
         "@types/passport-saml": "^1.1.3",
+        "@types/pem": "^1.9.6",
         "@types/uuid": "^8.3.0",
         "@types/ws": "^8.5.3",
         "@types/xml2js": "^0.4.7",
@@ -57,6 +58,7 @@
         "class-validator": "^0.14.0",
         "compression": "^1.7.4",
         "cookie-parser": "^1.4.5",
+        "crypto-js": "^4.1.1",
         "kafkajs": "^2.2.4",
         "lodash": "^4.17.20",
         "mqtt": "^4.3.7",
@@ -67,6 +69,7 @@
         "passport-jwt": "^4.0.0",
         "passport-local": "^1.0.0",
         "passport-saml": "^3.2.4",
+        "pem": "^1.14.7",
         "pg": "^8.5.1",
         "protobufjs": "^6.11.3",
         "reflect-metadata": "^0.1.13",
@@ -75,7 +78,7 @@
         "swagger-ui-express": "^4.1.5",
         "typeorm": "^0.3.10",
         "uuid": "^8.3.2",
-        "vm2": "^3.9.11",
+        "vm2": "^3.9.17",
         "wait-for-expect": "^3.0.2"
     },
     "devDependencies": {
@@ -87,6 +90,7 @@
         "@types/compression": "^1.7.0",
         "@types/cookie-parser": "^1.4.2",
         "@types/cron": "^1.7.2",
+        "@types/crypto-js": "^4.1.1",
         "@types/express": "^4.17.9",
         "@types/geojson": "^7946.0.7",
         "@types/kafkajs": "^1.9.0",

package.json

@@ -24,7 +24,7 @@ import {
     ApiUnauthorizedResponse,
 } from "@nestjs/swagger";
 
-import { ComposeAuthGuard } from '@auth/compose-auth.guard';
+import { ComposeAuthGuard } from "@auth/compose-auth.guard";
 import { Read, ApplicationAdmin } from "@auth/roles.decorator";
 import { RolesGuard } from "@auth/roles.guard";
 import { CreateIoTDevicePayloadDecoderDataTargetConnectionDto } from "@dto/create-iot-device-payload-decoder-data-target-connection.dto";
@@ -36,7 +36,10 @@ import { ListAllEntitiesDto } from "@dto/list-all-entities.dto";
 import { UpdateIoTDevicePayloadDecoderDataTargetConnectionDto as UpdateConnectionDto } from "@dto/update-iot-device-payload-decoder-data-target-connection.dto";
 import { IoTDevicePayloadDecoderDataTargetConnection } from "@entities/iot-device-payload-decoder-data-target-connection.entity";
 import { ErrorCodes } from "@enum/error-codes.enum";
-import { checkIfUserHasAccessToApplication, ApplicationAccessScope } from "@helpers/security-helper";
+import {
+    checkIfUserHasAccessToApplication,
+    ApplicationAccessScope,
+} from "@helpers/security-helper";
 import { IoTDevicePayloadDecoderDataTargetConnectionService } from "@services/device-management/iot-device-payload-decoder-data-target-connection.service";
 import { IoTDeviceService } from "@services/device-management/iot-device.service";
 import { AuditLog } from "@services/audit-log.service";
@@ -184,7 +187,11 @@ export class IoTDevicePayloadDecoderDataTargetConnectionController {
     ) {
         const iotDevices = await this.iotDeviceService.findManyByIds(ids);
         iotDevices.forEach(x => {
-            checkIfUserHasAccessToApplication(req, x.application.id, ApplicationAccessScope.Write);
+            checkIfUserHasAccessToApplication(
+                req,
+                x.application.id,
+                ApplicationAccessScope.Write
+            );
         });
     }
 
@@ -231,7 +238,11 @@ export class IoTDevicePayloadDecoderDataTargetConnectionController {
         const newIotDevice = await this.iotDeviceService.findOne(
             updateDto.iotDeviceIds[0]
         );
-        checkIfUserHasAccessToApplication(req, newIotDevice.application.id, ApplicationAccessScope.Write);
+        checkIfUserHasAccessToApplication(
+            req,
+            newIotDevice.application.id,
+            ApplicationAccessScope.Write
+        );
         const oldConnection = await this.service.findOne(id);
         await this.checkUserHasWriteAccessToAllIotDevices(updateDto.iotDeviceIds, req);
         const oldIds = oldConnection.iotDevices.map(x => x.id);

src/controllers/admin-controller/iot-device-payload-decoder-data-target-connection.controller.ts

@@ -12,6 +12,7 @@ import {
     Post,
     Put,
     Req,
+    StreamableFile,
     UseGuards,
 } from "@nestjs/common";
 import {
@@ -34,7 +35,10 @@ import { LoRaWANDeviceWithChirpstackDataDto } from "@dto/lorawan-device-with-chi
 import { UpdateIoTDeviceDto } from "@dto/update-iot-device.dto";
 import { IoTDevice } from "@entities/iot-device.entity";
 import { ErrorCodes } from "@enum/error-codes.enum";
-import { checkIfUserHasAccessToApplication, ApplicationAccessScope } from "@helpers/security-helper";
+import {
+    ApplicationAccessScope,
+    checkIfUserHasAccessToApplication,
+} from "@helpers/security-helper";
 import { IoTDeviceService } from "@services/device-management/iot-device.service";
 import { SigFoxDeviceWithBackendDataDto } from "@dto/sigfox-device-with-backend-data.dto";
 import { CreateIoTDeviceDownlinkDto } from "@dto/create-iot-device-downlink.dto";
@@ -56,6 +60,8 @@ import {
 } from "@helpers/iot-device.helper";
 import { DeviceStatsResponseDto } from "@dto/chirpstack/device/device-stats.response.dto";
 import { GenericHTTPDevice } from "@entities/generic-http-device.entity";
+import { MQTTInternalBrokerDeviceDTO } from "@dto/mqtt-internal-broker-device.dto";
+import { MQTTExternalBrokerDeviceDTO } from "@dto/mqtt-external-broker-device.dto";
 
 @ApiTags("IoT Device")
 @Controller("iot-device")
@@ -80,7 +86,11 @@ export class IoTDeviceController {
         @Req() req: AuthenticatedRequest,
         @Param("id", new ParseIntPipe()) id: number
     ): Promise<
-        IoTDevice | LoRaWANDeviceWithChirpstackDataDto | SigFoxDeviceWithBackendDataDto
+        | IoTDevice
+        | LoRaWANDeviceWithChirpstackDataDto
+        | SigFoxDeviceWithBackendDataDto
+        | MQTTInternalBrokerDeviceDTO
+        | MQTTExternalBrokerDeviceDTO
     > {
         let result = undefined;
         try {
@@ -96,7 +106,11 @@ export class IoTDeviceController {
             throw new NotFoundException(ErrorCodes.IdDoesNotExists);
         }
 
-        checkIfUserHasAccessToApplication(req, result.application.id, ApplicationAccessScope.Read);
+        checkIfUserHasAccessToApplication(
+            req,
+            result.application.id,
+            ApplicationAccessScope.Read
+        );
 
         return result;
     }
@@ -120,7 +134,11 @@ export class IoTDeviceController {
         if (!device) {
             throw new NotFoundException(ErrorCodes.IdDoesNotExists);
         }
-        checkIfUserHasAccessToApplication(req, device.application.id, ApplicationAccessScope.Read);
+        checkIfUserHasAccessToApplication(
+            req,
+            device.application.id,
+            ApplicationAccessScope.Read
+        );
         if (device.type == IoTDeviceType.LoRaWAN) {
             return this.chirpstackDeviceService.getDownlinkQueue(
                 (device as LoRaWANDevice).deviceEUI
@@ -141,7 +159,11 @@ export class IoTDeviceController {
         @Param("id", new ParseIntPipe()) id: number
     ): Promise<DeviceStatsResponseDto[]> {
         const device = await this.iotDeviceService.findOne(id);
-        checkIfUserHasAccessToApplication(req, device.application.id, ApplicationAccessScope.Read);
+        checkIfUserHasAccessToApplication(
+            req,
+            device.application.id,
+            ApplicationAccessScope.Read
+        );
 
         return this.iotDeviceService.findStats(device);
     }
@@ -155,7 +177,11 @@ export class IoTDeviceController {
         @Body() createDto: CreateIoTDeviceDto
     ): Promise<IoTDevice> {
         try {
-            checkIfUserHasAccessToApplication(req, createDto.applicationId, ApplicationAccessScope.Write);
+            checkIfUserHasAccessToApplication(
+                req,
+                createDto.applicationId,
+                ApplicationAccessScope.Write
+            );
             const device = await this.iotDeviceService.create(createDto, req.user.userId);
             AuditLog.success(
                 ActionType.CREATE,
@@ -192,7 +218,11 @@ export class IoTDeviceController {
             if (!device) {
                 throw new NotFoundException();
             }
-            checkIfUserHasAccessToApplication(req, device?.application?.id, ApplicationAccessScope.Write);
+            checkIfUserHasAccessToApplication(
+                req,
+                device?.application?.id,
+                ApplicationAccessScope.Write
+            );
             const result = await this.downlinkService.createDownlink(dto, device);
             AuditLog.success(ActionType.CREATE, "Downlink", req.user.userId);
             return result;
@@ -217,10 +247,18 @@ export class IoTDeviceController {
             false
         );
         try {
-            checkIfUserHasAccessToApplication(req, oldIotDevice.application.id, ApplicationAccessScope.Write);
+            checkIfUserHasAccessToApplication(
+                req,
+                oldIotDevice.application.id,
+                ApplicationAccessScope.Write
+            );
             if (updateDto.applicationId !== oldIotDevice.application.id) {
                 // New application
-                checkIfUserHasAccessToApplication(req, updateDto.applicationId, ApplicationAccessScope.Write);
+                checkIfUserHasAccessToApplication(
+                    req,
+                    updateDto.applicationId,
+                    ApplicationAccessScope.Write
+                );
             }
         } catch (err) {
             AuditLog.fail(ActionType.UPDATE, IoTDevice.name, req.user.userId, id);
@@ -251,7 +289,13 @@ export class IoTDeviceController {
         @Body() createDto: CreateIoTDeviceBatchDto
     ): Promise<IotDeviceBatchResponseDto[]> {
         try {
-            createDto.data.forEach(createDto => checkIfUserHasAccessToApplication(req, createDto.applicationId, ApplicationAccessScope.Write));
+            createDto.data.forEach(createDto =>
+                checkIfUserHasAccessToApplication(
+                    req,
+                    createDto.applicationId,
+                    ApplicationAccessScope.Write
+                )
+            );
 
             const devices = await this.iotDeviceService.createMany(
                 createDto.data,
@@ -348,7 +392,11 @@ export class IoTDeviceController {
                 id,
                 false
             );
-            checkIfUserHasAccessToApplication(req, oldIotDevice?.application?.id, ApplicationAccessScope.Write);
+            checkIfUserHasAccessToApplication(
+                req,
+                oldIotDevice?.application?.id,
+                ApplicationAccessScope.Write
+            );
             const result = await this.iotDeviceService.delete(oldIotDevice);
             AuditLog.success(ActionType.DELETE, IoTDevice.name, req.user.userId, id);
             return new DeleteResponseDto(result.affected);
@@ -364,16 +412,24 @@ export class IoTDeviceController {
     async resetHttpDeviceApiKey(
         @Req() req: AuthenticatedRequest,
         @Param("id", new ParseIntPipe()) id: number
-    ): Promise<Pick<GenericHTTPDevice, 'apiKey'>> {
+    ): Promise<Pick<GenericHTTPDevice, "apiKey">> {
         try {
             const oldIotDevice = await this.iotDeviceService.findOne(id);
-            checkIfUserHasAccessToApplication(req, oldIotDevice?.application?.id, ApplicationAccessScope.Write);
+            checkIfUserHasAccessToApplication(
+                req,
+                oldIotDevice?.application?.id,
+                ApplicationAccessScope.Write
+            );
 
             if (oldIotDevice.type !== IoTDeviceType.GenericHttp) {
-                throw new BadRequestException("The requested device is not a generic HTTP device");
+                throw new BadRequestException(
+                    "The requested device is not a generic HTTP device"
+                );
             }
 
-            const result = await this.iotDeviceService.resetHttpDeviceApiKey(oldIotDevice as GenericHTTPDevice);
+            const result = await this.iotDeviceService.resetHttpDeviceApiKey(
+                oldIotDevice as GenericHTTPDevice
+            );
             AuditLog.success(ActionType.UPDATE, IoTDevice.name, req.user.userId, id);
             return {
                 apiKey: result.apiKey,
@@ -383,4 +439,28 @@ export class IoTDeviceController {
             throw err;
         }
     }
+
+    @Get("getDevicesMetadataCsv/:applicationId")
+    @ApiOperation({
+        summary: "Get csv containing metadata for all devices in an application",
+    })
+    @ApiBadRequestResponse()
+    async getDevicesMetadataCsv(
+        @Req() req: AuthenticatedRequest,
+        @Param("applicationId", new ParseIntPipe()) applicationId: number
+    ): Promise<StreamableFile> {
+        try {
+            checkIfUserHasAccessToApplication(
+                req,
+                applicationId,
+                ApplicationAccessScope.Read
+            );
+            const csvFile = await this.iotDeviceService.getDevicesMetadataCsv(
+                applicationId
+            );
+            return new StreamableFile(csvFile);
+        } catch (err) {
+            this.logger.error(err);
+        }
+    }
 }

src/controllers/admin-controller/iot-device.controller.ts

@@ -18,6 +18,8 @@ import { CreateLoRaWANSettingsDto } from "./create-lorawan-settings.dto";
 import { CreateSigFoxSettingsDto } from "./create-sigfox-settings.dto";
 import { IsMetadataJson } from "@helpers/is-metadata-json.validator";
 import { nameof } from "@helpers/type-helper";
+import { CreateMqttInternalBrokerSettingsDto } from "@dto/create-mqtt-internal-broker-settings.dto";
+import { CreateMqttExternalBrokerSettingsDto } from "@dto/create-mqtt-external-broker-settings.dto";
 
 export class CreateIoTDeviceDto {
     @ApiProperty({ required: true })
@@ -71,14 +73,26 @@ export class CreateIoTDeviceDto {
     deviceModelId?: number;
 
     @ApiProperty({ required: false })
-    @ValidateIf(o => o.type == IoTDeviceType.LoRaWAN)
+    @ValidateIf(o => o.type === IoTDeviceType.LoRaWAN)
     @ValidateNested({ each: true })
     @Type(() => CreateLoRaWANSettingsDto)
     lorawanSettings?: CreateLoRaWANSettingsDto;
 
     @ApiProperty({ required: false })
-    @ValidateIf(o => o.type == IoTDeviceType.SigFox)
+    @ValidateIf(o => o.type === IoTDeviceType.SigFox)
     @ValidateNested({ each: true })
     @Type(() => CreateSigFoxSettingsDto)
     sigfoxSettings?: CreateSigFoxSettingsDto;
+
+    @ApiProperty({ required: false })
+    @ValidateIf(o => o.type === IoTDeviceType.MQTTInternalBroker)
+    @ValidateNested({ each: true })
+    @Type(() => CreateMqttInternalBrokerSettingsDto)
+    mqttInternalBrokerSettings?: CreateMqttInternalBrokerSettingsDto;
+
+    @ApiProperty({ required: false })
+    @ValidateIf(o => o.type === IoTDeviceType.MQTTExternalBroker)
+    @ValidateNested({ each: true })
+    @Type(() => CreateMqttExternalBrokerSettingsDto)
+    mqttExternalBrokerSettings?: CreateMqttExternalBrokerSettingsDto;
 }

src/entities/dto/create-iot-device.dto.ts

@@ -1,5 +1,13 @@
 import { ApiProperty, PickType } from "@nestjs/swagger";
-import { IsHexadecimal, IsIn, IsInt, IsNumber, IsOptional, IsString, Length, Matches, Min, ValidateIf } from "class-validator";
+import {
+    IsHexadecimal,
+    IsInt,
+    IsNumber,
+    IsString,
+    Length,
+    Min,
+    ValidateIf,
+} from "class-validator";
 
 import { ActivationType } from "@enum/lorawan-activation-type.enum";
 
@@ -37,7 +45,7 @@ export class CreateLoRaWANSettingsDto extends PickType(ChirpstackDeviceContentsD
     @ValidateIf((o: CreateLoRaWANSettingsDto) => o.activationType == ActivationType.ABP)
     @IsNumber()
     @IsInt()
-    @Min(0)    
+    @Min(0)
     fCntUp?: number;
 
     @ApiProperty({ required: false })