IOT-1157: Fetch all gateways at once instead of pagination

Author: VirtualSatai

src/controllers/admin-controller/chirpstack/chirpstack-gateway.controller.ts

@@ -35,6 +35,7 @@ import { checkIfUserHasWriteAccessToOrganization } from "@helpers/security-helpe
 import { AuthenticatedRequest } from "@dto/internal/authenticated-request";
 import { AuditLog } from "@services/audit-log.service";
 import { ActionType } from "@entities/audit-log-entry";
+import { ChirpstackGetAll } from "@dto/chirpstack/chirpstack-get-all.dto";
 
 @ApiTags("Chirpstack")
 @Controller("chirpstack/gateway")
@@ -87,11 +88,9 @@ export class ChirpstackGatewayController {
     @ApiOperation({ summary: "List all Chirpstack gateways" })
     @Read()
     async getAll(
-        @Query() query?: ChirpstackPaginatedListDto
+        @Query() query?: ChirpstackGetAll
     ): Promise<ListAllGatewaysResponseDto> {
-        return await this.chirpstackGatewayService.listAllPaginated(
-            query.limit,
-            query.offset,
+        return await this.chirpstackGatewayService.getAll(
             query.organizationId
         );
     }

src/entities/dto/chirpstack/chirpstack-get-all.dto.ts

@@ -0,0 +1,6 @@
+import { PickType } from "@nestjs/swagger";
+import { ChirpstackPaginatedListDto } from "./chirpstack-paginated-list.dto";
+
+export class ChirpstackGetAll extends PickType(ChirpstackPaginatedListDto, [
+    "organizationId",
+]) {}

src/services/chirpstack/chirpstack-gateway.service.ts

@@ -25,6 +25,7 @@ import { GatewayContentsDto } from "@dto/chirpstack/gateway-contents.dto";
 import * as _ from "lodash";
 import { AuthenticatedRequest } from "@dto/internal/authenticated-request";
 import { checkIfUserHasWriteAccessToOrganization } from "@helpers/security-helper";
+import { GatewayResponseDto } from "@dto/chirpstack/gateway-response.dto";
 
 @Injectable()
 export class ChirpstackGatewayService extends GenericChirpstackConfigurationService {
@@ -78,25 +79,42 @@ export class ChirpstackGatewayService extends GenericChirpstackConfigurationServ
         return tags;
     }
 
-    async listAllPaginated(
-        limit?: number,
-        offset?: number,
-        organizationId?: number
-    ): Promise<ListAllGatewaysResponseDto> {
-        // Default parameters if not set
-        if (!offset) {
-            offset = 0;
-        }
-        if (!limit) {
-            limit = 100;
+    async getAll(organizationId?: number): Promise<ListAllGatewaysResponseDto> {
+        const limit = 1000;
+        let allResults: GatewayResponseDto[] = [];
+        let totalCount = 0;
+        let lastResults;
+        do {
+            // Default parameters if not set
+            lastResults = await this.getAllWithPagination<ListAllGatewaysResponseDto>(
+                "gateways",
+                limit,
+                allResults.length
+            );
+            allResults = _.union(allResults, lastResults.result);
+            totalCount = lastResults.totalCount;
+        } while (totalCount > allResults.length && lastResults.result.length > 0);
+
+        await this.enrichWithOrganizationId(allResults);
+        if (organizationId !== undefined) {
+            const filteredResults = _.filter(allResults, x => {
+                return x.internalOrganizationId === +organizationId;
+            });
+            return {
+                result: filteredResults,
+                totalCount: filteredResults.length,
+            };
         }
-        const results = await this.getAllWithPagination<ListAllGatewaysResponseDto>(
-            "gateways",
-            limit,
-            offset
-        );
+
+        return {
+            result: allResults,
+            totalCount: totalCount,
+        };
+    }
+
+    private async enrichWithOrganizationId(results: GatewayResponseDto[]) {
         await Promise.all(
-            results.result.map(async x => {
+            results.map(async x => {
                 try {
                     const gw = await this.getOne(x.id);
                     x.internalOrganizationId = gw.gateway.internalOrganizationId;
@@ -109,17 +127,6 @@ export class ChirpstackGatewayService extends GenericChirpstackConfigurationServ
                 }
             })
         );
-        if (organizationId !== undefined) {
-            const filteredResults = _.filter(results.result, x => {
-                return x.internalOrganizationId === +organizationId;
-            });
-            return {
-                result: filteredResults,
-                totalCount: filteredResults.length,
-            };
-        }
-
-        return results;
     }
 
     async getOne(gatewayId: string): Promise<SingleGatewayResponseDto> {

test/e2e/chirpstack/chirpstack-gateway-configuration.e2e-spec.ts

@@ -81,7 +81,7 @@ describe("ChirpstackGatewayController (e2e)", () => {
     beforeEach(async () => {
         await clearDatabase();
         // Delete all gateways created in E2E tests:
-        const existing = await service.listAllPaginated(1000, 0);
+        const existing = await service.getAll(1000, 0);
         existing.result.forEach(async element => {
             if (element.name.startsWith(gatewayNamePrefix)) {
                 // Logger.debug(`Found ${element.name}, deleting.`);

test/e2e/crud/search.e2e-spec.ts

@@ -93,7 +93,7 @@ describe("SearchController (e2e)", () => {
         await clearDatabase();
 
         // Delete all gateways created in E2E tests:
-        const existing = await chirpstackGatewayService.listAllPaginated(1000, 0);
+        const existing = await chirpstackGatewayService.getAll(1000, 0);
         existing.result.forEach(async element => {
             if (element.name.startsWith(gatewayNamePrefix)) {
                 // Logger.debug(`Found ${element.name}, deleting.`);

test/e2e/generate_data_for_loadtest.py

@@ -9,8 +9,14 @@
     "Authorization": f"Bearer {JWT}"
 }
 
+
 def random_mac():
-    return "02:00:00:%02x:%02x:%02x" % (random.randint(0, 255), random.randint(0, 255), random.randint(0, 255))
+    return "02:00:00:%02x:%02x:%02x" % (random.randint(0, 255), random.randint(0, 255), random.randint(0, 255)
+                                        )
+
+
+def random_mac_16_no_seperator():
+    return "0201020100%02x%02x%02x" % (random.randint(0, 255), random.randint(0, 255), random.randint(0, 255))
 
 
 def generate_application():
@@ -41,8 +47,37 @@ def generate_iot_device():
     return id
 
 
+def generate_gateway():
+    id = random_mac_16_no_seperator()
+    dto = {
+        "gateway": {
+            "boards": [],
+            "description": PREFIX,
+            "discoveryEnabled": False,
+            "id": id,
+            "location": {
+                "longitude": random.uniform(10, 12),
+                "latitude": random.uniform(55, 57),
+                "altitude": random.uniform(10, 100),
+            },
+            "metadata": {},
+            "name": f"{PREFIX}-gw-{id}",
+            "tagsString": "{}"
+        },
+        "organizationId": 1
+    }
+    res = requests.post(
+        'http://[::1]:3000/api/v1/chirpstack/gateway', json=dto, headers=headers)
+    id = res.json()
+    print(f"made gateway: {id}")
+    return id
+
+
 for i in range(100):
     applications_made.append(generate_application())
 
 for i in range(10000):
     generate_iot_device()
+
+for i in range(100):
+    generate_gateway()