Add data source selection to population calculation

Calculating population in accessibility maps will now ask to enter a data source of the zones type.
The population function also returns a second parameter which indicates how much of the polygon is covered by the zones in the data source, and will warn the user when it is below 99% (for example, if the accessibility map is calculated just outside of the data source's border).

Author: GabrielBruno24

docs/APIv1/accessibilityMap.yml

@@ -85,6 +85,10 @@ accessibilityMapRequest:
       type: boolean
       description: Whether to calculate the population of the polygon and include the count in the result. Defaults to false.
       example: true
+    dataSourceId: 
+      type: string
+      description: UUID of the data source that contains the population data we want to use. Does nothing if calculatePopulation is set to false. Defaults to undefined.
+      example: 042ab131-0c2a-4ab6-b4a6-5b05d216d7dc
 
 
 accessibilityMapResponseSuccess:
@@ -224,6 +228,7 @@ accessibilityMapResultResponsePolygons:
             required:
               - durationSeconds
               - areaSqM
+              - populationData
             properties:
               durationSeconds: 
                 type: number
@@ -245,11 +250,21 @@ accessibilityMapResultResponsePolygons:
                   type: integer
                 description: The number of POIs within the accessibility polygon (if any), separated by detailed categories. The keys are the names of the detailed categories defined in the PlaceDetailedCategory type, and the values are integers.
                 example: {'service_bank': 8, 'service_other': 2, 'restaurant_restaurant': 5}
-              population:
-                type: number
-                nullable: true
-                description: The population inside the result polygon. Is null if calculatePopulation was set to false or the polygon does not cover any of the zones in the census db table.
-                example: 102457
+              populationData:
+                type: object
+                required:
+                  - population
+                  - dataSourceAreaRatio
+                properties:
+                  population:
+                    type: number
+                    nullable: true
+                    description: The population inside the result polygon. Is null if calculatePopulation was set to false or the polygon does not cover any of the zones in the census db table (distinct from 0, which is for a population that's actually calculated to be 0 based on the data available).
+                    example: 102457
+                  dataSourceAreaRatio:
+                    type: number
+                    description: The portion, from 0 to 1, of how much of the polygon is covered by the zones included in the population data source (for example, it might be less than 1 when the source contains only data for Canada and the polygon is on the US-Canada border).
+                    example: 0.67327
 
 accessibilityMapResponseBadRequest:
   type: string

locales/en/transit.json

@@ -546,6 +546,9 @@
         "SeeDetailedCategories": "See detailed categories",
         "ShowEmptyDetailedCategories": "Show empty detailed categories",
         "AccessiblePOIsPopup": "The data used to determine accessible Points of interest was downloaded from OpenStreetMap and imported into the database. The results may be inaccurate if the accessibility polygon is not contained entirely within the imported area.",
+        "DataSourceWarningPopup": "Warning: The population data from this data source covers only {{percentage}}% of the accessibility polygon. This may be due to it intersecting a body of water or the border of the region imported by this source.",
+        "DataSourceNullWarningPopup": "Warning: The population data from this data source does not cover any of the accessibility polygon. This may be due to it being contained in a body of water or outside the region imported by this source.",
+        "DataSourceNullWarningComparisonPopup": "Warning: The population data from this data source does not cover any of the accessibility polygon for at least one of the scenarios. This may be due to it being contained in a body of water or outside the region imported by this source.",
         "Scenario": "Scenario",
         "DataSource": "Data source",
         "CompareWithScenario": "Compare with scenario",
@@ -581,6 +584,7 @@
         "WithAlternatives": "Calculate with alternatives",
         "CalculatePois": "Calculate with points of interest",
         "CalculatePopulation": "Calculate with the population",
+        "PopulationDataSourceSelect": "Select a data source to use to calculate the population",
         "Detailed": "Detailed results with steps",
         "WithGeometries": "Include route geometries (geojson)",
         "ReverseOD": "Reverse",
@@ -688,7 +692,8 @@
             "InvalidOdTripsDataSource": "Invalid OD calculations data source.",
             "ErrorCalculatingLocation": "Error calculating location {{id}}",
             "UserDiskQuotaReached": "Maximum allowed disk space has been reached. Please delete old tasks to delete their files.",
-            "RoutingParametersInvalidForBatch": "Routing parameters are invalid. Make sure the scenario is set and times have a valid value."
+            "RoutingParametersInvalidForBatch": "Routing parameters are invalid. Make sure the scenario is set and times have a valid value.",
+            "DataSourceIsMissing": "The data source is required."
         },
         "actions": {
             "walking": "Walk",

locales/fr/transit.json

@@ -546,6 +546,9 @@
         "SeeDetailedCategories": "Voir les catégories détaillées",
         "ShowEmptyDetailedCategories": "Montrer les catégories détaillées vides",
         "AccessiblePOIsPopup": "Les données utilisées pour calculer les lieux d'activité accessibles ont été téléchargées de OpenStreetMap et importées dans la base de données. Les résultats peuvent être inexacts si le polygone d'accessibilité n'est pas contenu entièrement dans la surface importée.",
+        "DataSourceWarningPopup": "Attention: Les données de population de cette source de données ne couvrent que {{percentage}}% du polygone d'accessibilité. Cela peut être parce qu'il chevauche une étendue d'eau ou la frontière de la région importée par cette source.",
+        "DataSourceNullWarningPopup": "Attention: Les données de population de cette source de données ne couvrent aucune partie du polygone d'accessibilité. Cela peut être parce qu'il est contenu dans une étendue d'eau ou en dehors de la région importée par cette source.",
+        "DataSourceNullWarningComparisonPopup": "Attention: Les données de population de cette source de données ne couvrent aucune partie du polygone d'accessibilité pour au moins un des scénarios. Cela peut être parce qu'il est contenu dans une étendue d'eau ou en dehors de la région importée par cette source.",
         "Scenario": "Scénario",
         "DataSource": "Source de données",
         "CompareWithScenario": "Comparer avec scénario",
@@ -581,6 +584,7 @@
         "WithAlternatives": "Calcul avec alternatives",
         "CalculatePois": "Calcul avec lieux d'activité",
         "CalculatePopulation": "Calcul avec la population",
+        "PopulationDataSourceSelect": "Sélectionnez une source de données pour calculer la population",
         "Detailed": "Résultats détaillés avec étapes",
         "WithGeometries": "Inclure les géométries des parcours (geojson)",
         "ReverseOD": "Inverser",
@@ -688,7 +692,8 @@
             "InvalidOdTripsDataSource": "Source de données de calculs OD invalide.",
             "ErrorCalculatingLocation": "Erreur de calcul pour le lieu {{id}}",
             "UserDiskQuotaReached": "L'espace disque maximal permis a été atteint. Veuillez supprimer des tâches passées pour en supprimer les fichiers",
-            "RoutingParametersInvalidForBatch": "Les paramètres pour le calcul de chemin sont invalides. Assurez-vous que le scénario est bien configuré et que les temps sont valides."
+            "RoutingParametersInvalidForBatch": "Les paramètres pour le calcul de chemin sont invalides. Assurez-vous que le scénario est bien configuré et que les temps sont valides.",
+            "DataSourceIsMissing": "La source de données est requise."
         },
         "actions": {
             "walking": "Marche",

packages/transition-backend/src/api/__tests__/public.routes.test.ts

@@ -700,7 +700,7 @@ describe('Testing API endpoints', () => {
                             otherProperty: 'foo',
                             accessiblePlacesCountByCategory: calculatePoisAndPopulation ? { 'service': 10 } : undefined,
                             accessiblePlacesCountByDetailedCategory: calculatePoisAndPopulation ? { 'service_other': 4, 'service_bank': 6 } : undefined,
-                            population: calculatePoisAndPopulation ? 500 : null
+                            populationData: calculatePoisAndPopulation ? { population: 500, dataSourceAreaRatio: 1 } : { population: null, dataSourceAreaRatio: 0 }
                         }
                     }]
                 },
@@ -729,11 +729,11 @@ describe('Testing API endpoints', () => {
                                 areaSqM: 1000,
                                 accessiblePlacesCountByCategory: { 'service': 10 },
                                 accessiblePlacesCountByDetailedCategory: { 'service_other': 4, 'service_bank': 6 },
-                                population: 500
+                                populationData: { population: 500, dataSourceAreaRatio: 1 }
                             } : {
                                 durationSeconds: 900,
                                 areaSqM: 1000,
-                                population: null
+                                populationData: { population: null, dataSourceAreaRatio: 0 }
                             }
                         }]
                     }

packages/transition-backend/src/api/public/AccessibilityMapAPIResponse.ts

@@ -33,6 +33,7 @@ type AccessibilityMapAPIQueryResponse = {
     walkingSpeedMps: number;
     calculatePois: boolean;
     calculatePopulation: boolean;
+    dataSourceId?: string;
 };
 
 type AccessibilityMapAPIResultResponse = {
@@ -47,6 +48,7 @@ type AccessibilityMapAPIResultResponse = {
                 areaSqM: number;
                 accessiblePlacesCountByCategory?: { [key in PlaceCategory]: number };
                 accessiblePlacesCountByDetailedCategory?: { [key in PlaceDetailedCategory]: number };
+                populationData: { population: number | null; dataSourceAreaRatio: number };
             };
         }>;
     };
@@ -90,7 +92,8 @@ export default class AccessibilityMapAPIResponse extends APIResponseBase<
             maxTransferTravelTimeSeconds: queryParams.maxTransferTravelTimeSeconds!,
             walkingSpeedMps: queryParams.walkingSpeedMps!,
             calculatePois: queryParams.calculatePois!,
-            calculatePopulation: queryParams.calculatePopulation!
+            calculatePopulation: queryParams.calculatePopulation!,
+            dataSourceId: queryParams.dataSourceId || undefined
         };
     }
 
@@ -111,7 +114,7 @@ export default class AccessibilityMapAPIResponse extends APIResponseBase<
                                 accessiblePlacesCountByCategory: feature.properties!.accessiblePlacesCountByCategory,
                                 accessiblePlacesCountByDetailedCategory:
                                       feature.properties!.accessiblePlacesCountByDetailedCategory,
-                                population: feature.properties!.population
+                                populationData: feature.properties!.populationData
                             }
                         }))
                     }

packages/transition-backend/src/models/db/__tests__/census.db.test.ts

@@ -8,29 +8,45 @@ import { v4 as uuidV4 } from 'uuid';
 import knex from 'chaire-lib-backend/lib/config/shared/db.config';
 
 import dbQueries from '../census.db.queries';
+import dataSourceDbQueries from 'chaire-lib-backend/lib/models/db/dataSources.db.queries';
 import zonesDbQueries from 'chaire-lib-backend/lib/models/db/zones.db.queries';
 import { Zone as ObjectClass } from 'chaire-lib-common/lib/services/zones/Zone';
 import { Polygon } from 'geojson';
+import { DataSourceAttributes } from 'chaire-lib-common/lib/services/dataSource/DataSource';
+
+const dataSourceId = uuidV4();
+const dataSourceAttributes = {
+    id: dataSourceId,
+    type: 'zones',
+    shortname: 'TDS',
+    name: 'Test datasource',
+    data: {}
+};
 
 // The two zones are squares that are side by side
 const id1 = uuidV4();
 const newObjectAttributes = {  
     id: id1,
     internal_id: 'test',
+    dataSourceId: dataSourceId,
     geography: { type: 'Polygon' as const, coordinates: [ [ [-73, 45], [-73, 46], [-72, 46], [-72, 45], [-73, 45] ] ] }
 };
 
 const id2 = uuidV4();
 const newObjectAttributes2 = {
     id: id2,
     internal_id: 'test2',
+    dataSourceId: dataSourceId,
     geography: { type: 'Polygon' as const, coordinates: [[[-72, 45], [-72, 46], [-71, 46], [-71, 45], [-72, 45]]]}
 };
 
 beforeAll(async () => {
     jest.setTimeout(10000);
     await dbQueries.truncate();
     await zonesDbQueries.truncate();
+    await dataSourceDbQueries.truncate();
+
+    await dataSourceDbQueries.create(dataSourceAttributes as DataSourceAttributes);
     const newObject = new ObjectClass(newObjectAttributes, true);
     await zonesDbQueries.create(newObject.attributes);
     const newObject2 = new ObjectClass(newObjectAttributes2, true);
@@ -40,6 +56,7 @@ beforeAll(async () => {
 afterAll(async() => {
     await dbQueries.truncate();
     await zonesDbQueries.truncate();
+    await dataSourceDbQueries.truncate();
     await knex.destroy();
 });
 
@@ -83,8 +100,24 @@ describe('census', () => {
             coordinates: [[[-73, 45], [-73, 46], [-71.5, 46], [-71.5, 45], [-73, 45]]]
         };
 
-        const population = await dbQueries.getPopulationInPolygon(polygon);
-        expect(population).toEqual(123456 + 500 / 2);
+        const population = await dbQueries.getPopulationInPolygon(polygon, dataSourceId);
+        expect(population.population).toEqual(123456 + 500 / 2);
+        expect(population.dataSourceAreaRatio).toEqual(1);
+
+    });
+
+    test('population partially outside data source', async () => {
+
+        // Half of the polygon covers half the first zone and half of it is outside.
+        // Thus, the population should be half the first zone with a dataSourceAreaRatio of 0.5
+        const polygon: Polygon = {
+            type: 'Polygon',
+            coordinates: [[[-73.5, 45], [-73.5, 46], [-72.5, 46], [-72.5, 45], [-73.5, 45]]]
+        };
+
+        const population = await dbQueries.getPopulationInPolygon(polygon, dataSourceId);
+        expect(population.population).toEqual(123456 / 2);
+        expect(population.dataSourceAreaRatio).toEqual(0.5);
 
     });
 
@@ -95,8 +128,22 @@ describe('census', () => {
             coordinates: [[[0, 0], [0, 1], [1, 1], [1, 0], [0, 0]]]
         };
 
-        const population = await dbQueries.getPopulationInPolygon(polygon);
-        expect(population).toBeNull();
+        const population = await dbQueries.getPopulationInPolygon(polygon, dataSourceId);
+        expect(population.population).toBeNull();
+        expect(population.dataSourceAreaRatio).toEqual(0);
+
+    });
+
+    test('try to get population with data source not in db', async () => {
+
+        const polygon: Polygon = {
+            type: 'Polygon',
+            coordinates: [[[-73, 45], [-73, 46], [-71.5, 46], [-71.5, 45], [-73, 45]]]
+        };
+
+        const population = await dbQueries.getPopulationInPolygon(polygon, uuidV4());
+        expect(population.population).toBeNull();
+        expect(population.dataSourceAreaRatio).toEqual(0);
 
     });
 });

packages/transition-backend/src/models/db/census.db.queries.ts

@@ -63,28 +63,40 @@ const addPopulationBatch = async (inputArray: { internalId: string; population:
 };
 
 const getPopulationInPolygon = async (
-    accessibilityPolygon: GeoJSON.MultiPolygon | GeoJSON.Polygon
-): Promise<number | null> => {
+    accessibilityPolygon: GeoJSON.MultiPolygon | GeoJSON.Polygon,
+    dataSourceId: string
+): Promise<{ population: number | null; dataSourceAreaRatio: number }> => {
     try {
-        // Find all the zones that intersect the input polygon, and fetch their population, area, and the area of the intersection.
+        // Find all the zones that intersect the input polygon from a particular data source, and fetch their population, area, and the area of the intersection.
         // We multiply the population of each zone by the ratio between the area  of its intersection with the input polygon and its total area, to estimate the true population of zones that aren't entirely contained within the input polygon.
-        // TO avoid division by zero in the edge case of a degenerate zone with no area, we use the NULLIF function.
+        // To avoid division by zero in the edge case of a degenerate zone with no area, we use the NULLIF function.
+        // We also get the ratio of the area of the zones in the polygon that have data to the area of the polygon.
+        // When we get a polygon for which we have complete data, we expect that number to be 1 or very close to it, however it will be lower if data is missing from that area (for example, if we input a polygon that's over the Canada-US border while using a data source that only has data for Canada).
         const populationResponse = await knex.raw(
             `
                 SELECT COUNT(*) as row_count,
                 ROUND(SUM(
                     population * 
                     ST_AREA(ST_INTERSECTION(ST_GeomFromGeoJSON(:polygon), geography::geometry)) / 
                     NULLIF(ST_AREA(geography::geometry), 0)
-                )) as weighted_population
+                )) as weighted_population,
+                (
+                    ST_AREA(ST_INTERSECTION(ST_UNION(geography::geometry), ST_GeomFromGeoJSON(:polygon))) /
+                    NULLIF(ST_AREA(ST_GeomFromGeoJSON(:polygon)), 0)
+                ) as data_source_area_ratio
                 FROM ${tableName} c JOIN ${parentTable} z ON c.zone_id = z.id
-                WHERE ST_INTERSECTS(geography::geometry, ST_GeomFromGeoJSON(:polygon));
+                WHERE z.data_source_id = :dataSourceId 
+                AND ST_INTERSECTS(geography::geometry, ST_GeomFromGeoJSON(:polygon));
             `,
-            { polygon: JSON.stringify(accessibilityPolygon) }
+            { polygon: JSON.stringify(accessibilityPolygon), dataSourceId }
         );
 
         const result = populationResponse.rows[0];
-        return Number(result.row_count) === 0 ? null : Number(result.weighted_population);
+
+        return {
+            population: Number(result.row_count) === 0 ? null : Number(result.weighted_population),
+            dataSourceAreaRatio: result.data_source_area_ratio === null ? 0 : Number(result.data_source_area_ratio)
+        };
     } catch (error) {
         throw new TrError(`Problem getting population (knex error: ${error})`, 'CSDB0003', 'ProblemGettingPopulation');
     }