fix: handle environment documentation pagination

sdk/index.ts

@@ -329,7 +329,7 @@ export class Flagsmith {
         url: string,
         method: string,
         body?: { [key: string]: any }
-    ): Promise<any> {
+    ): Promise<{ response: Response; data: any }> {
         const headers: { [key: string]: any } = { 'Content-Type': 'application/json' };
         if (this.environmentKey) {
             headers['X-Environment-Key'] = this.environmentKey as string;
@@ -363,7 +363,7 @@ export class Flagsmith {
             );
         }
 
-        return data.json();
+        return { response: data, data: await data.json() };
     }
 
     /**
@@ -393,8 +393,52 @@ export class Flagsmith {
         if (!this.environmentUrl) {
             throw new Error('`apiUrl` argument is missing or invalid.');
         }
-        const environment_data = await this.getJSONResponse(this.environmentUrl, 'GET');
-        return buildEnvironmentModel(environment_data);
+        const startTime = Date.now();
+        const documents: any[] = [];
+        let url = this.environmentUrl;
+        let loggedWarning = false;
+
+        while (true) {
+            try {
+                if (!loggedWarning) {
+                    const elapsedMs = Date.now() - startTime;
+                    if (elapsedMs > this.environmentRefreshIntervalSeconds * 1000) {
+                        this.logger.warn(
+                            `Environment document retrieval exceeded the polling interval of ${this.environmentRefreshIntervalSeconds} seconds.`
+                        );
+                        loggedWarning = true;
+                    }
+                }
+
+                const { response, data } = await this.getJSONResponse(url, 'GET');
+
+                documents.push(data);
+
+                const linkHeader = response.headers.get('link');
+                if (linkHeader) {
+                    const nextMatch = linkHeader.match(/<([^>]+)>;\s*rel="next"/);
+
+                    if (nextMatch) {
+                        const relativeUrl = decodeURIComponent(nextMatch[1]);
+                        url = new URL(relativeUrl, this.apiUrl).href;
+
+                        continue;
+                    }
+                }
+                break;
+            } catch (error) {
+                throw error;
+            }
+        }
+
+        // Compile the document
+        const compiledDocument = documents[0];
+        for (let i = 1; i < documents.length; i++) {
+            compiledDocument.identity_overrides = compiledDocument.identity_overrides || [];
+            compiledDocument.identity_overrides.push(...(documents[i].identity_overrides || []));
+        }
+
+        return buildEnvironmentModel(compiledDocument);
     }
 
     private async getEnvironmentFlagsFromDocument(): Promise<Flags> {
@@ -444,7 +488,7 @@ export class Flagsmith {
         if (!this.environmentFlagsUrl) {
             throw new Error('`apiUrl` argument is missing or invalid.');
         }
-        const apiFlags = await this.getJSONResponse(this.environmentFlagsUrl, 'GET');
+        const { data: apiFlags } = await this.getJSONResponse(this.environmentFlagsUrl, 'GET');
         const flags = Flags.fromAPIFlags({
             apiFlags: apiFlags,
             analyticsProcessor: this.analyticsProcessor,
@@ -465,7 +509,7 @@ export class Flagsmith {
             throw new Error('`apiUrl` argument is missing or invalid.');
         }
         const data = generateIdentitiesData(identifier, traits, transient);
-        const jsonResponse = await this.getJSONResponse(this.identitiesUrl, 'POST', data);
+        const { data: jsonResponse } = await this.getJSONResponse(this.identitiesUrl, 'POST', data);
         const flags = Flags.fromAPIFlags({
             apiFlags: jsonResponse['flags'],
             analyticsProcessor: this.analyticsProcessor,

tests/engine/unit/engine.test.ts

@@ -12,7 +12,6 @@ import {
     environmentWithSegmentOverride,
     feature1,
     getEnvironmentFeatureStateForFeature,
-    getEnvironmentFeatureStateForFeatureByName,
     identity,
     identityInSegment,
     segmentConditionProperty,

tests/sdk/flagsmith.test.ts

@@ -40,6 +40,160 @@ test('test_update_environment_sets_environment', async () => {
     expect(await flg.getEnvironment()).toStrictEqual(model);
 });
 
+test('test_update_environment_handles_paginated_document', async () => {
+    type EnvDocumentMockResponse = {
+        responseHeader: string | null;
+        page: any;
+    };
+
+    const createMockFetch = (pages: EnvDocumentMockResponse[]) => {
+        let callCount = 0;
+        return vi.fn((url: string, options?: RequestInit) => {
+            if (url.includes('/environment-document')) {
+                const document = envDocumentMockResponse[callCount];
+                if (document) {
+                    callCount++;
+
+                    const responseHeaders: Record<string, string> = {};
+
+                    if (document.responseHeader) {
+                        responseHeaders['Link'] = `<${document.responseHeader}>; rel="next"`;
+                    }
+
+                    return Promise.resolve(
+                        new Response(JSON.stringify(document.page), {
+                            status: 200,
+                            headers: responseHeaders
+                        })
+                    );
+                }
+            }
+            return Promise.resolve(new Response('unknown url ' + url, { status: 404 }));
+        });
+    };
+
+    const envDocumentMockResponse: EnvDocumentMockResponse[] = [
+        {
+            responseHeader: '/api/v1/environment-document?page=2',
+            page: {
+                id: 1,
+                api_key: 'test-key',
+                project: {
+                    id: 1,
+                    name: 'test',
+                    organisation: {
+                        id: 1,
+                        name: 'Test Org',
+                        feature_analytics: false,
+                        persist_trait_data: true,
+                        stop_serving_flags: false
+                    },
+                    hide_disabled_flags: false,
+                    segments: []
+                },
+                feature_states: [
+                    {
+                        feature_state_value: 'first_page_feature_state',
+                        multivariate_feature_state_values: [],
+                        django_id: 81027,
+                        feature: {
+                            id: 15058,
+                            type: 'STANDARD',
+                            name: 'string_feature'
+                        },
+                        enabled: false
+                    },
+                    {
+                        feature_state_value: 'second_page_feature_state',
+                        multivariate_feature_state_values: [],
+                        django_id: 81027,
+                        feature: {
+                            id: 15058,
+                            type: 'STANDARD',
+                            name: 'string_feature'
+                        },
+                        enabled: false
+                    },
+                    {
+                        feature_state_value: 'third_page_feature_state',
+                        multivariate_feature_state_values: [],
+                        django_id: 81027,
+                        feature: {
+                            id: 15058,
+                            type: 'STANDARD',
+                            name: 'string_feature'
+                        },
+                        enabled: false
+                    }
+                ],
+                identity_overrides: [{ id: 1, identifier: 'user1' }]
+            }
+        },
+        {
+            responseHeader: '/api/v1/environment-document?page=3',
+            page: {
+                api_key: 'test-key',
+                project: {
+                    id: 1,
+                    name: 'test',
+                    organisation: {
+                        id: 1,
+                        name: 'Test Org',
+                        feature_analytics: false,
+                        persist_trait_data: true,
+                        stop_serving_flags: false
+                    },
+                    hide_disabled_flags: false,
+                    segments: []
+                },
+                feature_states: [],
+                identity_overrides: [{ id: 2, identifier: 'user2' }]
+            }
+        },
+        {
+            responseHeader: null,
+            page: {
+                api_key: 'test-key',
+                project: {
+                    id: 1,
+                    name: 'test',
+                    organisation: {
+                        id: 1,
+                        name: 'Test Org',
+                        feature_analytics: false,
+                        persist_trait_data: true,
+                        stop_serving_flags: false
+                    },
+                    hide_disabled_flags: false,
+                    segments: []
+                },
+                feature_states: [],
+                identity_overrides: [{ id: 2, identifier: 'user3' }]
+            }
+        }
+    ];
+
+    const flg = new Flagsmith({
+        environmentKey: 'ser.key',
+        enableLocalEvaluation: true,
+        fetch: createMockFetch(envDocumentMockResponse)
+    });
+
+    const environment = await flg.getEnvironment();
+
+    expect(environment.identityOverrides).toHaveLength(3);
+    expect(environment.identityOverrides[0].identifier).toBe('user1');
+    expect(environment.identityOverrides[1].identifier).toBe('user2');
+    expect(environment.identityOverrides[2].identifier).toBe('user3');
+    expect(environment.featureStates).toHaveLength(3);
+    expect(environment.featureStates[0].getValue()).toBe('first_page_feature_state');
+    expect(environment.featureStates[1].getValue()).toBe('second_page_feature_state');
+    expect(environment.featureStates[2].getValue()).toBe('third_page_feature_state');
+    expect(environment.project.name).toBe('test');
+    expect(environment.project.organisation.name).toBe('Test Org');
+    expect(environment.project.organisation.id).toBe(1);
+});
+
 test('test_set_agent_options', async () => {
     const agent = new Agent({});