diff --git a/internal/server/spanner/client.go b/internal/server/spanner/client.go index c9f391e53..11aa37819 100644 --- a/internal/server/spanner/client.go +++ b/internal/server/spanner/client.go @@ -25,6 +25,7 @@ import ( "cloud.google.com/go/spanner" pb "github.com/datacommonsorg/mixer/internal/proto" + pbv1 "github.com/datacommonsorg/mixer/internal/proto/v1" v2 "github.com/datacommonsorg/mixer/internal/server/v2" "github.com/datacommonsorg/mixer/internal/translator/types" "gopkg.in/yaml.v3" @@ -39,6 +40,8 @@ type SpannerClient interface { SearchNodes(ctx context.Context, query string, types []string) ([]*SearchNode, error) ResolveByID(ctx context.Context, nodes []string, in, out string) (map[string][]string, error) GetEventCollectionDate(ctx context.Context, placeID, eventType string) ([]string, error) + GetEventCollectionDcids(ctx context.Context, placeID, eventType, date string) ([]string, error) + GetEventCollection(ctx context.Context, req *pbv1.EventCollectionRequest) (*pbv1.EventCollection, error) Sparql(ctx context.Context, nodes []types.Node, queries []*types.Query, opts *types.QueryOptions) ([][]string, error) GetProvenanceSummary(ctx context.Context, ids []string) (map[string]map[string]*pb.StatVarSummary_ProvenanceSummary, error) Id() string diff --git a/internal/server/spanner/datasource.go b/internal/server/spanner/datasource.go index 1c2e06d62..a5596eeaa 100644 --- a/internal/server/spanner/datasource.go +++ b/internal/server/spanner/datasource.go @@ -27,6 +27,7 @@ import ( "github.com/datacommonsorg/mixer/internal/server/datasource" "github.com/datacommonsorg/mixer/internal/server/recon" v2 "github.com/datacommonsorg/mixer/internal/server/v2" + v2e "github.com/datacommonsorg/mixer/internal/server/v2/event" v3 "github.com/datacommonsorg/mixer/internal/server/v3" "github.com/datacommonsorg/mixer/internal/store/files" "github.com/datacommonsorg/mixer/internal/translator/sparql" @@ -347,6 +348,14 @@ func (sds *SpannerDataSource) Event(ctx context.Context, req *pbv2.EventRequest) return sds.handleEventCollectionDate(ctx, parsedReq) } + parsedReq, err := parseEventCollection(req, arcs) + if err != nil { + return nil, err + } + if parsedReq != nil { + return sds.handleEventCollection(ctx, parsedReq) + } + return nil, status.Errorf(codes.InvalidArgument, "unsupported event request property: %s", req.GetProperty()) } @@ -395,3 +404,62 @@ func (sds *SpannerDataSource) BulkVariableInfo(ctx context.Context, req *pbv1.Bu } return generateBulkVariableInfoResponse(metadata), nil } + +// parseEventCollection checks if the arcs match the pattern for EventCollection and returns the parsed request. +// Pattern: <-location{typeOf:EventType, date:Date, filter_prop:filter_val} +func parseEventCollection(req *pbv2.EventRequest, arcs []*v2.Arc) (*pbv1.EventCollectionRequest, error) { + if len(arcs) != 1 { + return nil, nil + } + arc := arcs[0] + if arc.Out || arc.SingleProp != "location" { + return nil, nil + } + typeOfs, ok := arc.Filter["typeOf"] + if !ok || len(typeOfs) == 0 { + return nil, fmt.Errorf("event collection requires 'typeOf' filter") + } + + res := &pbv1.EventCollectionRequest{ + EventType: typeOfs[0], + AffectedPlaceDcid: req.GetNode(), + } + + if dates, ok := arc.Filter["date"]; ok && len(dates) > 0 { + res.Date = dates[0] + } + + // Handle standard filters (e.g. area). + for k, v := range arc.Filter { + if k == "typeOf" || k == "date" { + continue + } + if len(v) != 1 { + return nil, fmt.Errorf("extra filter '%s' can only have one value", k) + } + spec, err := v2e.ParseEventCollectionFilter(k, v[0]) + if err != nil { + return nil, fmt.Errorf("invalid filter format for '%s': %v", k, err) + } + res.FilterProp = spec.Prop + res.FilterLowerLimit = spec.LowerLimit + res.FilterUpperLimit = spec.UpperLimit + res.FilterUnit = spec.Unit + break // V2 supports at most one extra filter + } + + return res, nil +} + +// handleEventCollection handles EventCollection requests. +func (sds *SpannerDataSource) handleEventCollection(ctx context.Context, req *pbv1.EventCollectionRequest) (*pbv2.EventResponse, error) { + collection, err := sds.client.GetEventCollection(ctx, req) + if err != nil { + return nil, fmt.Errorf("error getting event collection: %v", err) + } + + return &pbv2.EventResponse{ + EventCollection: collection, + }, nil + +} diff --git a/internal/server/spanner/golden/datasource/event_collection_lbr.json b/internal/server/spanner/golden/datasource/event_collection_lbr.json new file mode 100644 index 000000000..96efd9b2d --- /dev/null +++ b/internal/server/spanner/golden/datasource/event_collection_lbr.json @@ -0,0 +1,545 @@ +{ + "eventCollection": { + "events": [ + { + "places": [ + "Earth", + "africa", + "country/LBR" + ], + "geoLocations": [ + { + "point": { + "latitude": 6.91239, + "longitude": -11.3885 + } + } + ], + "dates": [ + "2020-10-09" + ], + "propVals": { + "area": { + "vals": [ + "SquareKilometer94.06721" + ] + }, + "endDate": { + "vals": [ + "2020-10-09" + ] + }, + "geoJsonCoordinates": { + "vals": [ + "{\"type\": \"Polygon\", \"coordinates\": [[[-11.433914835336811, 6.869123530871398], [-11.343105834556939, 6.8712958445565695], [-11.343105834556939, 6.95570216102805], [-11.433914835336813, 6.953503685766837], [-11.433914835336811, 6.869123530871398]]]}" + ] + }, + "name": { + "vals": [ + "FireEvent at Liberia on 2020-10-09" + ] + }, + "observationDate": { + "vals": [ + "2020-10-09" + ] + }, + "observationPeriod": { + "vals": [ + "P1D" + ] + } + }, + "dcid": "fireEvent/2020-10-09_0x0f09010000000000", + "provenanceId": "dc/base/NASA_VIIRSActiveFiresEvents" + }, + { + "places": [ + "Earth", + "africa", + "country/LBR" + ], + "geoLocations": [ + { + "point": { + "latitude": 5.93404, + "longitude": -10.03597 + } + } + ], + "dates": [ + "2020-10-14" + ], + "propVals": { + "area": { + "vals": [ + "SquareKilometer91.79452" + ] + }, + "endDate": { + "vals": [ + "2020-10-14" + ] + }, + "geoJsonCoordinates": { + "vals": [ + "{\"type\": \"Polygon\", \"coordinates\": [[[-10.080729817659453, 5.891543516844587], [-9.991225384867022, 5.893161056496378], [-9.991225384867024, 5.976566489231818], [-10.080729817659453, 5.97492638725516], [-10.080729817659453, 5.891543516844587]]]}" + ] + }, + "name": { + "vals": [ + "FireEvent at Liberia on 2020-10-14" + ] + }, + "observationDate": { + "vals": [ + "2020-10-14" + ] + }, + "observationPeriod": { + "vals": [ + "P1D" + ] + } + }, + "dcid": "fireEvent/2020-10-14_0x0fa0e70000000000", + "provenanceId": "dc/base/NASA_VIIRSActiveFiresEvents" + }, + { + "places": [ + "Earth", + "africa", + "country/LBR", + "wikidataId/Q1047921" + ], + "geoLocations": [ + { + "point": { + "latitude": 6.668, + "longitude": -11.02593 + } + } + ], + "dates": [ + "2020-10-15" + ], + "propVals": { + "area": { + "vals": [ + "SquareKilometer93.51209" + ] + }, + "endDate": { + "vals": [ + "2020-10-15" + ] + }, + "geoJsonCoordinates": { + "vals": [ + "{\"type\": \"Polygon\", \"coordinates\": [[[-11.071175792204036, 6.624915859903759], [-10.980700311326265, 6.626936443101766], [-10.980700311326265, 6.711110758715905], [-11.071175792204036, 6.709064978357141], [-11.071175792204036, 6.624915859903759]]]}" + ] + }, + "name": { + "vals": [ + "FireEvent at Grand Cape Mount County on 2020-10-15" + ] + }, + "observationDate": { + "vals": [ + "2020-10-15" + ] + }, + "observationPeriod": { + "vals": [ + "P1D" + ] + } + }, + "dcid": "fireEvent/2020-10-15_0x0f09bf0000000000", + "provenanceId": "dc/base/NASA_VIIRSActiveFiresEvents" + }, + { + "places": [ + "Earth", + "africa", + "country/LBR", + "wikidataId/Q3711106", + "wikidataId/Q868497" + ], + "geoLocations": [ + { + "point": { + "latitude": 7.13476, + "longitude": -8.9687 + } + } + ], + "dates": [ + "2020-10-19" + ], + "propVals": { + "area": { + "vals": [ + "SquareKilometer92.40039" + ] + }, + "endDate": { + "vals": [ + "2020-10-19" + ] + }, + "geoJsonCoordinates": { + "vals": [ + "{\"type\": \"Polygon\", \"coordinates\": [[[-9.01289263966051, 7.0912954005685185], [-8.924534343172422, 7.093003969627144], [-8.924534343172423, 7.178251858528139], [-9.012892639660508, 7.176523183651866], [-9.01289263966051, 7.0912954005685185]]]}" + ] + }, + "name": { + "vals": [ + "FireEvent at Leewehpea-Mahn District on 2020-10-19" + ] + }, + "observationDate": { + "vals": [ + "2020-10-19" + ] + }, + "observationPeriod": { + "vals": [ + "P1D" + ] + } + }, + "dcid": "fireEvent/2020-10-19_0x0fa5cf0000000000", + "provenanceId": "dc/base/NASA_VIIRSActiveFiresEvents" + }, + { + "places": [ + "Earth", + "africa", + "country/LBR" + ], + "geoLocations": [ + { + "point": { + "latitude": 7.63675, + "longitude": -9.50061 + } + } + ], + "dates": [ + "2020-10-19" + ], + "propVals": { + "area": { + "vals": [ + "SquareKilometer93.4783" + ] + }, + "endDate": { + "vals": [ + "2020-10-19" + ] + }, + "geoJsonCoordinates": { + "vals": [ + "{\"type\": \"Polygon\", \"coordinates\": [[[-9.54509739864785, 7.5929002105543235], [-9.45615499039193, 7.594849999197043], [-9.45615499039193, 7.680623968033509], [-9.545097398647847, 7.6786526859513415], [-9.54509739864785, 7.5929002105543235]]]}" + ] + }, + "name": { + "vals": [ + "FireEvent at Liberia on 2020-10-19" + ] + }, + "observationDate": { + "vals": [ + "2020-10-19" + ] + }, + "observationPeriod": { + "vals": [ + "P1D" + ] + } + }, + "dcid": "fireEvent/2020-10-19_0x0fa61f0000000000", + "provenanceId": "dc/base/NASA_VIIRSActiveFiresEvents" + }, + { + "places": [ + "Earth", + "africa", + "country/LBR", + "wikidataId/Q3031920", + "wikidataId/Q862608" + ], + "geoLocations": [ + { + "point": { + "latitude": 6.76423, + "longitude": -10.48462 + } + } + ], + "dates": [ + "2020-10-19" + ], + "propVals": { + "area": { + "vals": [ + "SquareKilometer93.24245" + ] + }, + "endDate": { + "vals": [ + "2020-10-19" + ] + }, + "geoJsonCoordinates": { + "vals": [ + "{\"type\": \"Polygon\", \"coordinates\": [[[-10.529611502460162, 6.721062747897819], [-10.439656018971009, 6.722997964061528], [-10.439656018971007, 6.807430721330982], [-10.529611502460162, 6.805471657239533], [-10.529611502460162, 6.721062747897819]]]}" + ] + }, + "name": { + "vals": [ + "FireEvent at Todee District on 2020-10-19" + ] + }, + "observationDate": { + "vals": [ + "2020-10-19" + ] + }, + "observationPeriod": { + "vals": [ + "P1D" + ] + } + }, + "dcid": "fireEvent/2020-10-19_0x0fa7650000000000", + "provenanceId": "dc/base/NASA_VIIRSActiveFiresEvents" + }, + { + "places": [ + "Earth", + "africa", + "country/LBR", + "wikidataId/Q27466632" + ], + "geoLocations": [ + { + "point": { + "latitude": 6.66595, + "longitude": -11.11645 + } + } + ], + "dates": [ + "2020-10-24" + ], + "propVals": { + "area": { + "vals": [ + "SquareKilometer93.57154" + ] + }, + "endDate": { + "vals": [ + "2020-10-24" + ] + }, + "geoJsonCoordinates": { + "vals": [ + "{\"type\": \"Polygon\", \"coordinates\": [[[-11.161735647774723, 6.622876980100405], [-11.071175792204036, 6.624915859903759], [-11.071175792204036, 6.709064978357141], [-11.161735647774721, 6.707000672797649], [-11.161735647774723, 6.622876980100405]]]}" + ] + }, + "name": { + "vals": [ + "FireEvent at Garwula on 2020-10-24" + ] + }, + "observationDate": { + "vals": [ + "2020-10-24" + ] + }, + "observationPeriod": { + "vals": [ + "P1D" + ] + } + }, + "dcid": "fireEvent/2020-10-24_0x0f09bd0000000000", + "provenanceId": "dc/base/NASA_VIIRSActiveFiresEvents" + }, + { + "places": [ + "Earth", + "africa", + "country/LBR", + "wikidataId/Q3711155", + "wikidataId/Q868497" + ], + "geoLocations": [ + { + "point": { + "latitude": 6.87619, + "longitude": -9.14562 + } + } + ], + "dates": [ + "2020-10-24" + ], + "propVals": { + "area": { + "vals": [ + "SquareKilometer184.52895" + ] + }, + "endDate": { + "vals": [ + "2020-10-24" + ] + }, + "geoJsonCoordinates": { + "vals": [ + "{\"type\": \"MultiPolygon\", \"coordinates\": [[[[-9.101349796048222, 6.834609098931902], [-9.189905212974633, 6.832924767497727], [-9.189905212974635, 6.917779558359327], [-9.101349796048222, 6.919484400964278], [-9.101349796048222, 6.834609098931902]]], [[[-9.101349796048224, 7.004470933809298], [-9.012892639660508, 7.006177833701461], [-9.012892639660508, 6.921170997740513], [-9.101349796048222, 6.919484400964278], [-9.101349796048224, 7.004470933809298]]]]}" + ] + }, + "name": { + "vals": [ + "FireEvent at Meinpea-Mahn District on 2020-10-24" + ] + }, + "observationDate": { + "vals": [ + "2020-10-24" + ] + }, + "observationPeriod": { + "vals": [ + "P1D" + ] + } + }, + "dcid": "fireEvent/2020-10-24_0x0fa4290000000000", + "provenanceId": "dc/base/NASA_VIIRSActiveFiresEvents" + }, + { + "places": [ + "Earth", + "africa", + "country/LBR", + "wikidataId/Q3710912", + "wikidataId/Q868497" + ], + "geoLocations": [ + { + "point": { + "latitude": 7.22005, + "longitude": -8.9687 + } + } + ], + "dates": [ + "2020-10-24" + ], + "propVals": { + "area": { + "vals": [ + "SquareKilometer92.50197" + ] + }, + "endDate": { + "vals": [ + "2020-10-24" + ] + }, + "geoJsonCoordinates": { + "vals": [ + "{\"type\": \"Polygon\", \"coordinates\": [[[-9.012892639660508, 7.176523183651866], [-8.924534343172423, 7.178251858528139], [-8.924534343172423, 7.263609456476478], [-9.012892639660508, 7.261860665238935], [-9.012892639660508, 7.176523183651866]]]}" + ] + }, + "name": { + "vals": [ + "FireEvent at Garr Bain District on 2020-10-24" + ] + }, + "observationDate": { + "vals": [ + "2020-10-24" + ] + }, + "observationPeriod": { + "vals": [ + "P1D" + ] + } + }, + "dcid": "fireEvent/2020-10-24_0x0fa5c50000000000", + "provenanceId": "dc/base/NASA_VIIRSActiveFiresEvents" + }, + { + "places": [ + "Earth", + "africa", + "country/LBR", + "wikidataId/Q3031879", + "wikidataId/Q877578" + ], + "geoLocations": [ + { + "point": { + "latitude": 7.03723, + "longitude": -9.5896 + } + } + ], + "dates": [ + "2020-10-24" + ], + "propVals": { + "area": { + "vals": [ + "SquareKilometer92.85048" + ] + }, + "endDate": { + "vals": [ + "2020-10-24" + ] + }, + "geoJsonCoordinates": { + "vals": [ + "{\"type\": \"Polygon\", \"coordinates\": [[[-9.634135027848398, 6.993840387817968], [-9.54509739864785, 6.995658664946421], [-9.54509739864785, 7.080651034016841], [-9.6341350278484, 7.078811114804452], [-9.634135027848398, 6.993840387817968]]]}" + ] + }, + "name": { + "vals": [ + "FireEvent at Suakoko District on 2020-10-24" + ] + }, + "observationDate": { + "vals": [ + "2020-10-24" + ] + }, + "observationPeriod": { + "vals": [ + "P1D" + ] + } + }, + "dcid": "fireEvent/2020-10-24_0x0fa65f0000000000", + "provenanceId": "dc/base/NASA_VIIRSActiveFiresEvents" + } + ], + "provenanceInfo": { + "dc/base/NASA_VIIRSActiveFiresEvents": { + "importName": "NASA_VIIRSActiveFiresEvents", + "provenanceUrl": "https://www.earthdata.nasa.gov/learn/find-data/near-real-time/firms/vnp14imgtdlnrt", + "domain": "nasa.gov" + } + } + } +} \ No newline at end of file diff --git a/internal/server/spanner/golden/datasource/event_collection_lbr_filtered.json b/internal/server/spanner/golden/datasource/event_collection_lbr_filtered.json new file mode 100644 index 000000000..b4b16ff70 --- /dev/null +++ b/internal/server/spanner/golden/datasource/event_collection_lbr_filtered.json @@ -0,0 +1,338 @@ +{ + "eventCollection": { + "events": [ + { + "places": [ + "Earth", + "africa", + "country/LBR", + "wikidataId/Q3711155", + "wikidataId/Q868497" + ], + "geoLocations": [ + { + "point": { + "latitude": 6.87619, + "longitude": -9.14562 + } + } + ], + "dates": [ + "2020-10-24" + ], + "propVals": { + "area": { + "vals": [ + "SquareKilometer184.52895" + ] + }, + "endDate": { + "vals": [ + "2020-10-24" + ] + }, + "geoJsonCoordinates": { + "vals": [ + "{\"type\": \"MultiPolygon\", \"coordinates\": [[[[-9.101349796048222, 6.834609098931902], [-9.189905212974633, 6.832924767497727], [-9.189905212974635, 6.917779558359327], [-9.101349796048222, 6.919484400964278], [-9.101349796048222, 6.834609098931902]]], [[[-9.101349796048224, 7.004470933809298], [-9.012892639660508, 7.006177833701461], [-9.012892639660508, 6.921170997740513], [-9.101349796048222, 6.919484400964278], [-9.101349796048224, 7.004470933809298]]]]}" + ] + }, + "name": { + "vals": [ + "FireEvent at Meinpea-Mahn District on 2020-10-24" + ] + }, + "observationDate": { + "vals": [ + "2020-10-24" + ] + }, + "observationPeriod": { + "vals": [ + "P1D" + ] + } + }, + "dcid": "fireEvent/2020-10-24_0x0fa4290000000000", + "provenanceId": "dc/base/NASA_VIIRSActiveFiresEvents" + }, + { + "places": [ + "Earth", + "africa", + "country/LBR", + "wikidataId/Q3711155", + "wikidataId/Q868497" + ], + "geoLocations": [ + { + "point": { + "latitude": 6.96111, + "longitude": -9.14562 + } + } + ], + "dates": [ + "2020-10-29" + ], + "propVals": { + "area": { + "vals": [ + "SquareKilometer184.68923" + ] + }, + "endDate": { + "vals": [ + "2020-10-30" + ] + }, + "geoJsonCoordinates": { + "vals": [ + "{\"type\": \"MultiPolygon\", \"coordinates\": [[[[-9.189905212974633, 7.002745568141814], [-9.101349796048224, 7.004470933809298], [-9.101349796048222, 6.919484400964278], [-9.189905212974635, 6.917779558359327], [-9.189905212974633, 7.002745568141814]]], [[[-9.189905212974633, 6.832924767497727], [-9.27855828820766, 6.831222371480414], [-9.27855828820766, 6.9160564308838435], [-9.189905212974635, 6.917779558359327], [-9.189905212974633, 6.832924767497727]]]]}" + ] + }, + "name": { + "vals": [ + "FireEvent at Meinpea-Mahn District on 2020-10-29" + ] + }, + "observationDate": { + "vals": [ + "2020-10-29", + "2020-10-30" + ] + }, + "observationPeriod": { + "vals": [ + "P2D" + ] + } + }, + "dcid": "fireEvent/2020-10-29_0x0fa42b0000000000", + "provenanceId": "dc/base/NASA_VIIRSActiveFiresEvents" + }, + { + "places": [ + "Earth", + "africa", + "country/LBR" + ], + "geoLocations": [ + { + "point": { + "latitude": 7.03166, + "longitude": -9.85714 + } + } + ], + "dates": [ + "2020-10-29" + ], + "propVals": { + "area": { + "vals": [ + "SquareKilometer185.96912" + ] + }, + "endDate": { + "vals": [ + "2020-10-30" + ] + }, + "geoJsonCoordinates": { + "vals": [ + "{\"type\": \"MultiPolygon\", \"coordinates\": [[[[-9.901813063435178, 6.988273240173706], [-9.901813063435178, 7.073177701895403], [-9.812493480789382, 7.075074485328742], [-9.812493480789382, 6.990147711575564], [-9.901813063435178, 6.988273240173706]]], [[[-9.72326726161636, 6.992003415497629], [-9.723267261616362, 6.907165189418345], [-9.812493480789382, 6.905331561283287], [-9.812493480789382, 6.990147711575564], [-9.72326726161636, 6.992003415497629]]]]}" + ] + }, + "name": { + "vals": [ + "FireEvent at Liberia on 2020-10-29" + ] + }, + "observationDate": { + "vals": [ + "2020-10-29", + "2020-10-30" + ] + }, + "observationPeriod": { + "vals": [ + "P2D" + ] + } + }, + "dcid": "fireEvent/2020-10-29_0x0fa6550000000000", + "provenanceId": "dc/base/NASA_VIIRSActiveFiresEvents" + }, + { + "places": [ + "Earth", + "africa", + "country/LBR", + "wikidataId/Q3031804", + "wikidataId/Q877578" + ], + "geoLocations": [ + { + "point": { + "latitude": 6.77187, + "longitude": -10.12552 + } + } + ], + "dates": [ + "2020-10-29" + ], + "propVals": { + "area": { + "vals": [ + "SquareKilometer185.83528" + ] + }, + "endDate": { + "vals": [ + "2020-10-29" + ] + }, + "geoJsonCoordinates": { + "vals": [ + "{\"type\": \"Polygon\", \"coordinates\": [[[-10.080729817659453, 6.646142102885027], [-10.17032573166423, 6.644302928060019], [-10.17032573166423, 6.728693790334738], [-10.17032573166423, 6.813196735755789], [-10.080729817659451, 6.815081787321996], [-10.080729817659453, 6.730555896401217], [-10.080729817659453, 6.646142102885027]]]}" + ] + }, + "name": { + "vals": [ + "FireEvent at Salala District on 2020-10-29" + ] + }, + "observationDate": { + "vals": [ + "2020-10-29" + ] + }, + "observationPeriod": { + "vals": [ + "P1D" + ] + } + }, + "dcid": "fireEvent/2020-10-29_0x0fa7170000000000", + "provenanceId": "dc/base/NASA_VIIRSActiveFiresEvents" + }, + { + "places": [ + "Earth", + "africa", + "country/LBR", + "wikidataId/Q3031987", + "wikidataId/Q870130" + ], + "geoLocations": [ + { + "point": { + "latitude": 8.14251, + "longitude": -9.94651 + } + } + ], + "dates": [ + "2020-10-29" + ], + "propVals": { + "area": { + "vals": [ + "SquareKilometer188.83275" + ] + }, + "endDate": { + "vals": [ + "2020-10-31" + ] + }, + "geoJsonCoordinates": { + "vals": [ + "{\"type\": \"MultiPolygon\", \"coordinates\": [[[[-9.991225384867024, 8.012114400365196], [-10.080729817659453, 8.009928422929677], [-10.080729817659451, 8.096073968017157], [-9.991225384867024, 8.098282829280363], [-9.991225384867024, 8.012114400365196]]], [[[-9.991225384867022, 8.18455498758315], [-9.901813063435178, 8.186764746913918], [-9.901813063435178, 8.100469923343903], [-9.991225384867024, 8.098282829280363], [-9.991225384867022, 8.18455498758315]]]]}" + ] + }, + "name": { + "vals": [ + "FireEvent at Voinjama District on 2020-10-29" + ] + }, + "observationDate": { + "vals": [ + "2020-10-29", + "2020-10-31" + ] + }, + "observationPeriod": { + "vals": [ + "P3D" + ] + } + }, + "dcid": "fireEvent/2020-10-29_0x0fa85d0000000000", + "provenanceId": "dc/base/NASA_VIIRSActiveFiresEvents" + }, + { + "places": [ + "Earth", + "africa", + "country/LBR", + "wikidataId/Q3031920", + "wikidataId/Q862608" + ], + "geoLocations": [ + { + "point": { + "latitude": 6.68178, + "longitude": -10.39471 + } + } + ], + "dates": [ + "2020-10-30" + ], + "propVals": { + "area": { + "vals": [ + "SquareKilometer186.24469" + ] + }, + "endDate": { + "vals": [ + "2020-10-30" + ] + }, + "geoJsonCoordinates": { + "vals": [ + "{\"type\": \"Polygon\", \"coordinates\": [[[-10.34978946920158, 6.640570531019608], [-10.439656018971007, 6.6386772462403485], [-10.439656018971009, 6.722997964061528], [-10.439656018971007, 6.807430721330982], [-10.34978946920158, 6.809371234843888], [-10.34978946920158, 6.724914855847337], [-10.34978946920158, 6.640570531019608]]]}" + ] + }, + "name": { + "vals": [ + "FireEvent at Todee District on 2020-10-30" + ] + }, + "observationDate": { + "vals": [ + "2020-10-30" + ] + }, + "observationPeriod": { + "vals": [ + "P1D" + ] + } + }, + "dcid": "fireEvent/2020-10-30_0x0fa7690000000000", + "provenanceId": "dc/base/NASA_VIIRSActiveFiresEvents" + } + ], + "provenanceInfo": { + "dc/base/NASA_VIIRSActiveFiresEvents": { + "importName": "NASA_VIIRSActiveFiresEvents", + "provenanceUrl": "https://www.earthdata.nasa.gov/learn/find-data/near-real-time/firms/vnp14imgtdlnrt", + "domain": "nasa.gov" + } + } + } +} \ No newline at end of file diff --git a/internal/server/spanner/golden/datasource_test.go b/internal/server/spanner/golden/datasource_test.go index 5e7e48bf4..ff9de4d03 100644 --- a/internal/server/spanner/golden/datasource_test.go +++ b/internal/server/spanner/golden/datasource_test.go @@ -22,6 +22,7 @@ import ( "github.com/datacommonsorg/mixer/internal/maps" pb "github.com/datacommonsorg/mixer/internal/proto" + pbv1 "github.com/datacommonsorg/mixer/internal/proto/v1" pbv2 "github.com/datacommonsorg/mixer/internal/proto/v2" "github.com/datacommonsorg/mixer/internal/server/datasources" "github.com/datacommonsorg/mixer/internal/server/spanner" @@ -67,6 +68,12 @@ func (m *mockSpannerClient) GetProvenanceSummary(ctx context.Context, ids []stri func (m *mockSpannerClient) GetEventCollectionDate(ctx context.Context, placeID, eventType string) ([]string, error) { return nil, nil } +func (m *mockSpannerClient) GetEventCollectionDcids(ctx context.Context, placeID, eventType, date string) ([]string, error) { + return nil, nil +} +func (m *mockSpannerClient) GetEventCollection(ctx context.Context, req *pbv1.EventCollectionRequest) (*pbv1.EventCollection, error) { + return nil, nil +} func (m *mockSpannerClient) Id() string { return "mock" } func (m *mockSpannerClient) Start() {} func (m *mockSpannerClient) Close() {} @@ -295,15 +302,33 @@ func TestSpannerEvent(t *testing.T) { }, goldenFile: "event_collection_date_lbr.json", }, + { + req: &pbv2.EventRequest{ + Node: "country/LBR", + Property: "<-location{typeOf:FireEvent,date:2020-10}", + }, + goldenFile: "event_collection_lbr.json", + }, + { + req: &pbv2.EventRequest{ + Node: "country/LBR", + Property: "<-location{typeOf:FireEvent,date:2020-10,area:100#200#SquareKilometer}", + }, + goldenFile: "event_collection_lbr_filtered.json", + }, } { got, err := ds.Event(ctx, c.req) if err != nil { t.Fatalf("Event error (%v): %v", c.goldenFile, err) } + // Trim to 10 events to avoid very large golden files. + if got.EventCollection != nil && len(got.EventCollection.Events) > 10 { + got.EventCollection.Events = got.EventCollection.Events[:10] + } if test.GenerateGolden { test.UpdateProtoGolden(got, goldenDir, c.goldenFile) - return + continue } var want pbv2.EventResponse diff --git a/internal/server/spanner/golden/query/get_event_collection.json b/internal/server/spanner/golden/query/get_event_collection.json new file mode 100644 index 000000000..93b3de51d --- /dev/null +++ b/internal/server/spanner/golden/query/get_event_collection.json @@ -0,0 +1,563 @@ +{ + "events": [ + { + "places": [ + "Earth", + "africa", + "country/LBR" + ], + "geo_locations": [ + { + "Geo": { + "Point": { + "latitude": 6.91239, + "longitude": -11.3885 + } + } + } + ], + "dates": [ + "2020-10-09" + ], + "prop_vals": { + "area": { + "vals": [ + "SquareKilometer94.06721" + ] + }, + "endDate": { + "vals": [ + "2020-10-09" + ] + }, + "geoJsonCoordinates": { + "vals": [ + "{\"type\": \"Polygon\", \"coordinates\": [[[-11.433914835336811, 6.869123530871398], [-11.343105834556939, 6.8712958445565695], [-11.343105834556939, 6.95570216102805], [-11.433914835336813, 6.953503685766837], [-11.433914835336811, 6.869123530871398]]]}" + ] + }, + "name": { + "vals": [ + "FireEvent at Liberia on 2020-10-09" + ] + }, + "observationDate": { + "vals": [ + "2020-10-09" + ] + }, + "observationPeriod": { + "vals": [ + "P1D" + ] + } + }, + "dcid": "fireEvent/2020-10-09_0x0f09010000000000", + "provenance_id": "dc/base/NASA_VIIRSActiveFiresEvents" + }, + { + "places": [ + "Earth", + "africa", + "country/LBR" + ], + "geo_locations": [ + { + "Geo": { + "Point": { + "latitude": 5.93404, + "longitude": -10.03597 + } + } + } + ], + "dates": [ + "2020-10-14" + ], + "prop_vals": { + "area": { + "vals": [ + "SquareKilometer91.79452" + ] + }, + "endDate": { + "vals": [ + "2020-10-14" + ] + }, + "geoJsonCoordinates": { + "vals": [ + "{\"type\": \"Polygon\", \"coordinates\": [[[-10.080729817659453, 5.891543516844587], [-9.991225384867022, 5.893161056496378], [-9.991225384867024, 5.976566489231818], [-10.080729817659453, 5.97492638725516], [-10.080729817659453, 5.891543516844587]]]}" + ] + }, + "name": { + "vals": [ + "FireEvent at Liberia on 2020-10-14" + ] + }, + "observationDate": { + "vals": [ + "2020-10-14" + ] + }, + "observationPeriod": { + "vals": [ + "P1D" + ] + } + }, + "dcid": "fireEvent/2020-10-14_0x0fa0e70000000000", + "provenance_id": "dc/base/NASA_VIIRSActiveFiresEvents" + }, + { + "places": [ + "Earth", + "africa", + "country/LBR", + "wikidataId/Q1047921" + ], + "geo_locations": [ + { + "Geo": { + "Point": { + "latitude": 6.668, + "longitude": -11.02593 + } + } + } + ], + "dates": [ + "2020-10-15" + ], + "prop_vals": { + "area": { + "vals": [ + "SquareKilometer93.51209" + ] + }, + "endDate": { + "vals": [ + "2020-10-15" + ] + }, + "geoJsonCoordinates": { + "vals": [ + "{\"type\": \"Polygon\", \"coordinates\": [[[-11.071175792204036, 6.624915859903759], [-10.980700311326265, 6.626936443101766], [-10.980700311326265, 6.711110758715905], [-11.071175792204036, 6.709064978357141], [-11.071175792204036, 6.624915859903759]]]}" + ] + }, + "name": { + "vals": [ + "FireEvent at Grand Cape Mount County on 2020-10-15" + ] + }, + "observationDate": { + "vals": [ + "2020-10-15" + ] + }, + "observationPeriod": { + "vals": [ + "P1D" + ] + } + }, + "dcid": "fireEvent/2020-10-15_0x0f09bf0000000000", + "provenance_id": "dc/base/NASA_VIIRSActiveFiresEvents" + }, + { + "places": [ + "Earth", + "africa", + "country/LBR", + "wikidataId/Q3711106", + "wikidataId/Q868497" + ], + "geo_locations": [ + { + "Geo": { + "Point": { + "latitude": 7.13476, + "longitude": -8.9687 + } + } + } + ], + "dates": [ + "2020-10-19" + ], + "prop_vals": { + "area": { + "vals": [ + "SquareKilometer92.40039" + ] + }, + "endDate": { + "vals": [ + "2020-10-19" + ] + }, + "geoJsonCoordinates": { + "vals": [ + "{\"type\": \"Polygon\", \"coordinates\": [[[-9.01289263966051, 7.0912954005685185], [-8.924534343172422, 7.093003969627144], [-8.924534343172423, 7.178251858528139], [-9.012892639660508, 7.176523183651866], [-9.01289263966051, 7.0912954005685185]]]}" + ] + }, + "name": { + "vals": [ + "FireEvent at Leewehpea-Mahn District on 2020-10-19" + ] + }, + "observationDate": { + "vals": [ + "2020-10-19" + ] + }, + "observationPeriod": { + "vals": [ + "P1D" + ] + } + }, + "dcid": "fireEvent/2020-10-19_0x0fa5cf0000000000", + "provenance_id": "dc/base/NASA_VIIRSActiveFiresEvents" + }, + { + "places": [ + "Earth", + "africa", + "country/LBR" + ], + "geo_locations": [ + { + "Geo": { + "Point": { + "latitude": 7.63675, + "longitude": -9.50061 + } + } + } + ], + "dates": [ + "2020-10-19" + ], + "prop_vals": { + "area": { + "vals": [ + "SquareKilometer93.4783" + ] + }, + "endDate": { + "vals": [ + "2020-10-19" + ] + }, + "geoJsonCoordinates": { + "vals": [ + "{\"type\": \"Polygon\", \"coordinates\": [[[-9.54509739864785, 7.5929002105543235], [-9.45615499039193, 7.594849999197043], [-9.45615499039193, 7.680623968033509], [-9.545097398647847, 7.6786526859513415], [-9.54509739864785, 7.5929002105543235]]]}" + ] + }, + "name": { + "vals": [ + "FireEvent at Liberia on 2020-10-19" + ] + }, + "observationDate": { + "vals": [ + "2020-10-19" + ] + }, + "observationPeriod": { + "vals": [ + "P1D" + ] + } + }, + "dcid": "fireEvent/2020-10-19_0x0fa61f0000000000", + "provenance_id": "dc/base/NASA_VIIRSActiveFiresEvents" + }, + { + "places": [ + "Earth", + "africa", + "country/LBR", + "wikidataId/Q3031920", + "wikidataId/Q862608" + ], + "geo_locations": [ + { + "Geo": { + "Point": { + "latitude": 6.76423, + "longitude": -10.48462 + } + } + } + ], + "dates": [ + "2020-10-19" + ], + "prop_vals": { + "area": { + "vals": [ + "SquareKilometer93.24245" + ] + }, + "endDate": { + "vals": [ + "2020-10-19" + ] + }, + "geoJsonCoordinates": { + "vals": [ + "{\"type\": \"Polygon\", \"coordinates\": [[[-10.529611502460162, 6.721062747897819], [-10.439656018971009, 6.722997964061528], [-10.439656018971007, 6.807430721330982], [-10.529611502460162, 6.805471657239533], [-10.529611502460162, 6.721062747897819]]]}" + ] + }, + "name": { + "vals": [ + "FireEvent at Todee District on 2020-10-19" + ] + }, + "observationDate": { + "vals": [ + "2020-10-19" + ] + }, + "observationPeriod": { + "vals": [ + "P1D" + ] + } + }, + "dcid": "fireEvent/2020-10-19_0x0fa7650000000000", + "provenance_id": "dc/base/NASA_VIIRSActiveFiresEvents" + }, + { + "places": [ + "Earth", + "africa", + "country/LBR", + "wikidataId/Q27466632" + ], + "geo_locations": [ + { + "Geo": { + "Point": { + "latitude": 6.66595, + "longitude": -11.11645 + } + } + } + ], + "dates": [ + "2020-10-24" + ], + "prop_vals": { + "area": { + "vals": [ + "SquareKilometer93.57154" + ] + }, + "endDate": { + "vals": [ + "2020-10-24" + ] + }, + "geoJsonCoordinates": { + "vals": [ + "{\"type\": \"Polygon\", \"coordinates\": [[[-11.161735647774723, 6.622876980100405], [-11.071175792204036, 6.624915859903759], [-11.071175792204036, 6.709064978357141], [-11.161735647774721, 6.707000672797649], [-11.161735647774723, 6.622876980100405]]]}" + ] + }, + "name": { + "vals": [ + "FireEvent at Garwula on 2020-10-24" + ] + }, + "observationDate": { + "vals": [ + "2020-10-24" + ] + }, + "observationPeriod": { + "vals": [ + "P1D" + ] + } + }, + "dcid": "fireEvent/2020-10-24_0x0f09bd0000000000", + "provenance_id": "dc/base/NASA_VIIRSActiveFiresEvents" + }, + { + "places": [ + "Earth", + "africa", + "country/LBR", + "wikidataId/Q3711155", + "wikidataId/Q868497" + ], + "geo_locations": [ + { + "Geo": { + "Point": { + "latitude": 6.87619, + "longitude": -9.14562 + } + } + } + ], + "dates": [ + "2020-10-24" + ], + "prop_vals": { + "area": { + "vals": [ + "SquareKilometer184.52895" + ] + }, + "endDate": { + "vals": [ + "2020-10-24" + ] + }, + "geoJsonCoordinates": { + "vals": [ + "{\"type\": \"MultiPolygon\", \"coordinates\": [[[[-9.101349796048222, 6.834609098931902], [-9.189905212974633, 6.832924767497727], [-9.189905212974635, 6.917779558359327], [-9.101349796048222, 6.919484400964278], [-9.101349796048222, 6.834609098931902]]], [[[-9.101349796048224, 7.004470933809298], [-9.012892639660508, 7.006177833701461], [-9.012892639660508, 6.921170997740513], [-9.101349796048222, 6.919484400964278], [-9.101349796048224, 7.004470933809298]]]]}" + ] + }, + "name": { + "vals": [ + "FireEvent at Meinpea-Mahn District on 2020-10-24" + ] + }, + "observationDate": { + "vals": [ + "2020-10-24" + ] + }, + "observationPeriod": { + "vals": [ + "P1D" + ] + } + }, + "dcid": "fireEvent/2020-10-24_0x0fa4290000000000", + "provenance_id": "dc/base/NASA_VIIRSActiveFiresEvents" + }, + { + "places": [ + "Earth", + "africa", + "country/LBR", + "wikidataId/Q3710912", + "wikidataId/Q868497" + ], + "geo_locations": [ + { + "Geo": { + "Point": { + "latitude": 7.22005, + "longitude": -8.9687 + } + } + } + ], + "dates": [ + "2020-10-24" + ], + "prop_vals": { + "area": { + "vals": [ + "SquareKilometer92.50197" + ] + }, + "endDate": { + "vals": [ + "2020-10-24" + ] + }, + "geoJsonCoordinates": { + "vals": [ + "{\"type\": \"Polygon\", \"coordinates\": [[[-9.012892639660508, 7.176523183651866], [-8.924534343172423, 7.178251858528139], [-8.924534343172423, 7.263609456476478], [-9.012892639660508, 7.261860665238935], [-9.012892639660508, 7.176523183651866]]]}" + ] + }, + "name": { + "vals": [ + "FireEvent at Garr Bain District on 2020-10-24" + ] + }, + "observationDate": { + "vals": [ + "2020-10-24" + ] + }, + "observationPeriod": { + "vals": [ + "P1D" + ] + } + }, + "dcid": "fireEvent/2020-10-24_0x0fa5c50000000000", + "provenance_id": "dc/base/NASA_VIIRSActiveFiresEvents" + }, + { + "places": [ + "Earth", + "africa", + "country/LBR", + "wikidataId/Q3031879", + "wikidataId/Q877578" + ], + "geo_locations": [ + { + "Geo": { + "Point": { + "latitude": 7.03723, + "longitude": -9.5896 + } + } + } + ], + "dates": [ + "2020-10-24" + ], + "prop_vals": { + "area": { + "vals": [ + "SquareKilometer92.85048" + ] + }, + "endDate": { + "vals": [ + "2020-10-24" + ] + }, + "geoJsonCoordinates": { + "vals": [ + "{\"type\": \"Polygon\", \"coordinates\": [[[-9.634135027848398, 6.993840387817968], [-9.54509739864785, 6.995658664946421], [-9.54509739864785, 7.080651034016841], [-9.6341350278484, 7.078811114804452], [-9.634135027848398, 6.993840387817968]]]}" + ] + }, + "name": { + "vals": [ + "FireEvent at Suakoko District on 2020-10-24" + ] + }, + "observationDate": { + "vals": [ + "2020-10-24" + ] + }, + "observationPeriod": { + "vals": [ + "P1D" + ] + } + }, + "dcid": "fireEvent/2020-10-24_0x0fa65f0000000000", + "provenance_id": "dc/base/NASA_VIIRSActiveFiresEvents" + } + ], + "provenance_info": { + "dc/base/NASA_VIIRSActiveFiresEvents": { + "import_name": "NASA_VIIRSActiveFiresEvents", + "provenance_url": "https://www.earthdata.nasa.gov/learn/find-data/near-real-time/firms/vnp14imgtdlnrt", + "domain": "nasa.gov" + } + } +} \ No newline at end of file diff --git a/internal/server/spanner/golden/query/get_event_collection_date_flood.json b/internal/server/spanner/golden/query/get_event_collection_date_flood.json new file mode 100644 index 000000000..82272e4e8 --- /dev/null +++ b/internal/server/spanner/golden/query/get_event_collection_date_flood.json @@ -0,0 +1,123 @@ +[ + "2015-07", + "2015-08", + "2015-09", + "2015-10", + "2015-11", + "2015-12", + "2016-01", + "2016-02", + "2016-03", + "2016-04", + "2016-05", + "2016-06", + "2016-07", + "2016-08", + "2016-09", + "2016-10", + "2016-11", + "2016-12", + "2017-01", + "2017-02", + "2017-03", + "2017-04", + "2017-05", + "2017-06", + "2017-07", + "2017-08", + "2017-09", + "2017-10", + "2017-11", + "2017-12", + "2018-01", + "2018-02", + "2018-04", + "2018-05", + "2018-06", + "2018-07", + "2018-08", + "2018-09", + "2018-10", + "2018-11", + "2018-12", + "2019-01", + "2019-02", + "2019-03", + "2019-04", + "2019-05", + "2019-06", + "2019-07", + "2019-08", + "2019-09", + "2019-10", + "2019-11", + "2019-12", + "2020-01", + "2020-02", + "2020-03", + "2020-04", + "2020-05", + "2020-06", + "2020-07", + "2020-08", + "2020-09", + "2020-10", + "2020-11", + "2020-12", + "2021-01", + "2021-02", + "2021-03", + "2021-04", + "2021-05", + "2021-06", + "2021-07", + "2021-08", + "2021-09", + "2021-10", + "2021-11", + "2021-12", + "2022-01", + "2022-02", + "2022-03", + "2022-04", + "2022-05", + "2022-06", + "2022-07", + "2022-08", + "2022-09", + "2022-10", + "2022-11", + "2022-12", + "2023-01", + "2023-02", + "2023-03", + "2023-04", + "2023-05", + "2023-06", + "2023-07", + "2023-08", + "2023-09", + "2023-10", + "2023-11", + "2023-12", + "2024-01", + "2024-03", + "2024-04", + "2024-05", + "2024-06", + "2024-07", + "2024-08", + "2024-09", + "2024-10", + "2024-11", + "2024-12", + "2025-01", + "2025-02", + "2025-03", + "2025-05", + "2025-06", + "2025-07", + "2025-08", + "2025-09", + "2025-10" +] \ No newline at end of file diff --git a/internal/server/spanner/golden/query/get_event_collection_dcids.json b/internal/server/spanner/golden/query/get_event_collection_dcids.json new file mode 100644 index 000000000..f574147dd --- /dev/null +++ b/internal/server/spanner/golden/query/get_event_collection_dcids.json @@ -0,0 +1,31 @@ +[ + "fireEvent/2020-10-30_0x0fa7690000000000", + "fireEvent/2020-10-24_0x0fa5c50000000000", + "fireEvent/2020-10-14_0x0fa0e70000000000", + "fireEvent/2020-10-31_0x0fa3490000000000", + "fireEvent/2020-10-29_0x0fa7790000000000", + "fireEvent/2020-10-24_0x0fa84f0000000000", + "fireEvent/2020-10-29_0x0fa85d0000000000", + "fireEvent/2020-10-29_0x0fa6550000000000", + "fireEvent/2020-10-19_0x0fa7650000000000", + "fireEvent/2020-10-24_0x0f09bd0000000000", + "fireEvent/2020-10-30_0x0fa7550000000000", + "fireEvent/2020-10-29_0x0fa7170000000000", + "fireEvent/2020-10-30_0x0fa7910000000000", + "fireEvent/2020-10-09_0x0f09010000000000", + "fireEvent/2020-10-30_0x0f97810000000000", + "fireEvent/2020-10-26_0x0f09c50000000000", + "fireEvent/2020-10-26_0x0fa5d30000000000", + "fireEvent/2020-10-15_0x0f09bf0000000000", + "fireEvent/2020-10-19_0x0fa61f0000000000", + "fireEvent/2020-10-25_0x0fa43d0000000000", + "fireEvent/2020-10-29_0x0fa42b0000000000", + "fireEvent/2020-10-19_0x0fa5cf0000000000", + "fireEvent/2020-10-31_0x0fa75b0000000000", + "fireEvent/2020-10-31_0x0fa1250000000000", + "fireEvent/2020-10-30_0x0fa65f0000000000", + "fireEvent/2020-10-30_0x0fa0af0000000000", + "fireEvent/2020-10-24_0x0fa65f0000000000", + "fireEvent/2020-10-24_0x0fa4290000000000", + "fireEvent/2020-10-30_0x0fa6e30000000000" +] \ No newline at end of file diff --git a/internal/server/spanner/golden/query/get_event_collection_dcids_flood.json b/internal/server/spanner/golden/query/get_event_collection_dcids_flood.json new file mode 100644 index 000000000..db3aa74be --- /dev/null +++ b/internal/server/spanner/golden/query/get_event_collection_dcids_flood.json @@ -0,0 +1,32 @@ +[ + "floodEvent/2025-01_0x4884fd0000000000", + "floodEvent/2025-01_0x487a0d0000000000", + "floodEvent/2025-01_0x4864510000000000", + "floodEvent/2025-01_0x487c4f0000000000", + "floodEvent/2025-01_0x4877c10000000000", + "floodEvent/2025-01_0x4878f90000000000", + "floodEvent/2025-01_0x489e570000000000", + "floodEvent/2025-01_0x487bd50000000000", + "floodEvent/2025-01_0x894b5f0000000000", + "floodEvent/2025-01_0x48720b0000000000", + "floodEvent/2025-01_0x47d0a90000000000", + "floodEvent/2025-01_0x487bed0000000000", + "floodEvent/2025-01_0x486f830000000000", + "floodEvent/2025-01_0x48691f0000000000", + "floodEvent/2025-01_0x4876c90000000000", + "floodEvent/2025-01_0x48881f0000000000", + "floodEvent/2025-01_0x486f2d0000000000", + "floodEvent/2025-01_0x47d7690000000000", + "floodEvent/2025-01_0x486c190000000000", + "floodEvent/2025-01_0x47d8110000000000", + "floodEvent/2025-01_0x487dcf0000000000", + "floodEvent/2025-01_0x4876710000000000", + "floodEvent/2025-01_0x487c6d0000000000", + "floodEvent/2025-01_0x8eb2d50000000000", + "floodEvent/2025-01_0x48628b0000000000", + "floodEvent/2025-01_0x4876270000000000", + "floodEvent/2025-01_0x48770d0000000000", + "floodEvent/2025-01_0x487d9f0000000000", + "floodEvent/2025-01_0x487f170000000000", + "floodEvent/2025-01_0x14de290000000000" +] \ No newline at end of file diff --git a/internal/server/spanner/golden/query/get_event_collection_filtered.json b/internal/server/spanner/golden/query/get_event_collection_filtered.json new file mode 100644 index 000000000..19c576ced --- /dev/null +++ b/internal/server/spanner/golden/query/get_event_collection_filtered.json @@ -0,0 +1,348 @@ +{ + "events": [ + { + "places": [ + "Earth", + "africa", + "country/LBR", + "wikidataId/Q3711155", + "wikidataId/Q868497" + ], + "geo_locations": [ + { + "Geo": { + "Point": { + "latitude": 6.87619, + "longitude": -9.14562 + } + } + } + ], + "dates": [ + "2020-10-24" + ], + "prop_vals": { + "area": { + "vals": [ + "SquareKilometer184.52895" + ] + }, + "endDate": { + "vals": [ + "2020-10-24" + ] + }, + "geoJsonCoordinates": { + "vals": [ + "{\"type\": \"MultiPolygon\", \"coordinates\": [[[[-9.101349796048222, 6.834609098931902], [-9.189905212974633, 6.832924767497727], [-9.189905212974635, 6.917779558359327], [-9.101349796048222, 6.919484400964278], [-9.101349796048222, 6.834609098931902]]], [[[-9.101349796048224, 7.004470933809298], [-9.012892639660508, 7.006177833701461], [-9.012892639660508, 6.921170997740513], [-9.101349796048222, 6.919484400964278], [-9.101349796048224, 7.004470933809298]]]]}" + ] + }, + "name": { + "vals": [ + "FireEvent at Meinpea-Mahn District on 2020-10-24" + ] + }, + "observationDate": { + "vals": [ + "2020-10-24" + ] + }, + "observationPeriod": { + "vals": [ + "P1D" + ] + } + }, + "dcid": "fireEvent/2020-10-24_0x0fa4290000000000", + "provenance_id": "dc/base/NASA_VIIRSActiveFiresEvents" + }, + { + "places": [ + "Earth", + "africa", + "country/LBR", + "wikidataId/Q3711155", + "wikidataId/Q868497" + ], + "geo_locations": [ + { + "Geo": { + "Point": { + "latitude": 6.96111, + "longitude": -9.14562 + } + } + } + ], + "dates": [ + "2020-10-29" + ], + "prop_vals": { + "area": { + "vals": [ + "SquareKilometer184.68923" + ] + }, + "endDate": { + "vals": [ + "2020-10-30" + ] + }, + "geoJsonCoordinates": { + "vals": [ + "{\"type\": \"MultiPolygon\", \"coordinates\": [[[[-9.189905212974633, 7.002745568141814], [-9.101349796048224, 7.004470933809298], [-9.101349796048222, 6.919484400964278], [-9.189905212974635, 6.917779558359327], [-9.189905212974633, 7.002745568141814]]], [[[-9.189905212974633, 6.832924767497727], [-9.27855828820766, 6.831222371480414], [-9.27855828820766, 6.9160564308838435], [-9.189905212974635, 6.917779558359327], [-9.189905212974633, 6.832924767497727]]]]}" + ] + }, + "name": { + "vals": [ + "FireEvent at Meinpea-Mahn District on 2020-10-29" + ] + }, + "observationDate": { + "vals": [ + "2020-10-29", + "2020-10-30" + ] + }, + "observationPeriod": { + "vals": [ + "P2D" + ] + } + }, + "dcid": "fireEvent/2020-10-29_0x0fa42b0000000000", + "provenance_id": "dc/base/NASA_VIIRSActiveFiresEvents" + }, + { + "places": [ + "Earth", + "africa", + "country/LBR" + ], + "geo_locations": [ + { + "Geo": { + "Point": { + "latitude": 7.03166, + "longitude": -9.85714 + } + } + } + ], + "dates": [ + "2020-10-29" + ], + "prop_vals": { + "area": { + "vals": [ + "SquareKilometer185.96912" + ] + }, + "endDate": { + "vals": [ + "2020-10-30" + ] + }, + "geoJsonCoordinates": { + "vals": [ + "{\"type\": \"MultiPolygon\", \"coordinates\": [[[[-9.901813063435178, 6.988273240173706], [-9.901813063435178, 7.073177701895403], [-9.812493480789382, 7.075074485328742], [-9.812493480789382, 6.990147711575564], [-9.901813063435178, 6.988273240173706]]], [[[-9.72326726161636, 6.992003415497629], [-9.723267261616362, 6.907165189418345], [-9.812493480789382, 6.905331561283287], [-9.812493480789382, 6.990147711575564], [-9.72326726161636, 6.992003415497629]]]]}" + ] + }, + "name": { + "vals": [ + "FireEvent at Liberia on 2020-10-29" + ] + }, + "observationDate": { + "vals": [ + "2020-10-29", + "2020-10-30" + ] + }, + "observationPeriod": { + "vals": [ + "P2D" + ] + } + }, + "dcid": "fireEvent/2020-10-29_0x0fa6550000000000", + "provenance_id": "dc/base/NASA_VIIRSActiveFiresEvents" + }, + { + "places": [ + "Earth", + "africa", + "country/LBR", + "wikidataId/Q3031804", + "wikidataId/Q877578" + ], + "geo_locations": [ + { + "Geo": { + "Point": { + "latitude": 6.77187, + "longitude": -10.12552 + } + } + } + ], + "dates": [ + "2020-10-29" + ], + "prop_vals": { + "area": { + "vals": [ + "SquareKilometer185.83528" + ] + }, + "endDate": { + "vals": [ + "2020-10-29" + ] + }, + "geoJsonCoordinates": { + "vals": [ + "{\"type\": \"Polygon\", \"coordinates\": [[[-10.080729817659453, 6.646142102885027], [-10.17032573166423, 6.644302928060019], [-10.17032573166423, 6.728693790334738], [-10.17032573166423, 6.813196735755789], [-10.080729817659451, 6.815081787321996], [-10.080729817659453, 6.730555896401217], [-10.080729817659453, 6.646142102885027]]]}" + ] + }, + "name": { + "vals": [ + "FireEvent at Salala District on 2020-10-29" + ] + }, + "observationDate": { + "vals": [ + "2020-10-29" + ] + }, + "observationPeriod": { + "vals": [ + "P1D" + ] + } + }, + "dcid": "fireEvent/2020-10-29_0x0fa7170000000000", + "provenance_id": "dc/base/NASA_VIIRSActiveFiresEvents" + }, + { + "places": [ + "Earth", + "africa", + "country/LBR", + "wikidataId/Q3031987", + "wikidataId/Q870130" + ], + "geo_locations": [ + { + "Geo": { + "Point": { + "latitude": 8.14251, + "longitude": -9.94651 + } + } + } + ], + "dates": [ + "2020-10-29" + ], + "prop_vals": { + "area": { + "vals": [ + "SquareKilometer188.83275" + ] + }, + "endDate": { + "vals": [ + "2020-10-31" + ] + }, + "geoJsonCoordinates": { + "vals": [ + "{\"type\": \"MultiPolygon\", \"coordinates\": [[[[-9.991225384867024, 8.012114400365196], [-10.080729817659453, 8.009928422929677], [-10.080729817659451, 8.096073968017157], [-9.991225384867024, 8.098282829280363], [-9.991225384867024, 8.012114400365196]]], [[[-9.991225384867022, 8.18455498758315], [-9.901813063435178, 8.186764746913918], [-9.901813063435178, 8.100469923343903], [-9.991225384867024, 8.098282829280363], [-9.991225384867022, 8.18455498758315]]]]}" + ] + }, + "name": { + "vals": [ + "FireEvent at Voinjama District on 2020-10-29" + ] + }, + "observationDate": { + "vals": [ + "2020-10-29", + "2020-10-31" + ] + }, + "observationPeriod": { + "vals": [ + "P3D" + ] + } + }, + "dcid": "fireEvent/2020-10-29_0x0fa85d0000000000", + "provenance_id": "dc/base/NASA_VIIRSActiveFiresEvents" + }, + { + "places": [ + "Earth", + "africa", + "country/LBR", + "wikidataId/Q3031920", + "wikidataId/Q862608" + ], + "geo_locations": [ + { + "Geo": { + "Point": { + "latitude": 6.68178, + "longitude": -10.39471 + } + } + } + ], + "dates": [ + "2020-10-30" + ], + "prop_vals": { + "area": { + "vals": [ + "SquareKilometer186.24469" + ] + }, + "endDate": { + "vals": [ + "2020-10-30" + ] + }, + "geoJsonCoordinates": { + "vals": [ + "{\"type\": \"Polygon\", \"coordinates\": [[[-10.34978946920158, 6.640570531019608], [-10.439656018971007, 6.6386772462403485], [-10.439656018971009, 6.722997964061528], [-10.439656018971007, 6.807430721330982], [-10.34978946920158, 6.809371234843888], [-10.34978946920158, 6.724914855847337], [-10.34978946920158, 6.640570531019608]]]}" + ] + }, + "name": { + "vals": [ + "FireEvent at Todee District on 2020-10-30" + ] + }, + "observationDate": { + "vals": [ + "2020-10-30" + ] + }, + "observationPeriod": { + "vals": [ + "P1D" + ] + } + }, + "dcid": "fireEvent/2020-10-30_0x0fa7690000000000", + "provenance_id": "dc/base/NASA_VIIRSActiveFiresEvents" + } + ], + "provenance_info": { + "dc/base/NASA_VIIRSActiveFiresEvents": { + "import_name": "NASA_VIIRSActiveFiresEvents", + "provenance_url": "https://www.earthdata.nasa.gov/learn/find-data/near-real-time/firms/vnp14imgtdlnrt", + "domain": "nasa.gov" + } + } +} \ No newline at end of file diff --git a/internal/server/spanner/golden/query/get_provenance_summary.json b/internal/server/spanner/golden/query/get_provenance_summary.json index 3dce996f1..6ac4f610f 100644 --- a/internal/server/spanner/golden/query/get_provenance_summary.json +++ b/internal/server/spanner/golden/query/get_provenance_summary.json @@ -41,6 +41,107 @@ ], "observation_count": 2547106, "time_series_count": 220438 + }, + { + "import_name": "CensusACS5YearSurvey_AggCountry", + "series_summary": [ + { + "series_key": { + "measurement_method": "CensusACS5yrSurvey", + "is_dc_aggregate": true + }, + "earliest_date": "2011", + "latest_date": "2023", + "place_type_summary": { + "Country": {} + }, + "observation_count": 13, + "time_series_count": 1 + } + ], + "observation_count": 13, + "time_series_count": 1 + }, + { + "import_name": "CensusACS5YearSurvey_SubjectTables_S1702", + "series_summary": [ + { + "series_key": { + "measurement_method": "CensusACS5yrSurveySubjectTable" + }, + "earliest_date": "2010", + "latest_date": "2023", + "place_type_summary": { + "AdministrativeArea1": {}, + "AdministrativeArea2": {}, + "AdministrativeArea4": {}, + "AdministrativeArea5": {}, + "Borough": {}, + "CensusCoreBasedStatisticalArea": {}, + "CensusCountyDivision": {}, + "CensusDivision": {}, + "CensusTract": {}, + "CensusZipCodeTabulationArea": {}, + "City": {}, + "CongressionalDistrict": {}, + "Country": {}, + "County": {}, + "ElementarySchoolDistrict": {}, + "HighSchoolDistrict": {}, + "Neighborhood": {}, + "Place": {}, + "ProvisionalNode": {}, + "SchoolDistrict": {}, + "State": {}, + "Town": {}, + "Village": {} + }, + "observation_count": 2702027, + "time_series_count": 220002 + } + ], + "observation_count": 2702027, + "time_series_count": 220002 + }, + { + "import_name": "CensusACS5YearSurvey_SubjectTables_S1901", + "series_summary": [ + { + "series_key": { + "measurement_method": "CensusACS5yrSurveySubjectTable" + }, + "earliest_date": "2010", + "latest_date": "2023", + "place_type_summary": { + "AdministrativeArea1": {}, + "AdministrativeArea2": {}, + "AdministrativeArea4": {}, + "AdministrativeArea5": {}, + "Borough": {}, + "CensusCoreBasedStatisticalArea": {}, + "CensusCountyDivision": {}, + "CensusTract": {}, + "CensusZipCodeTabulationArea": {}, + "City": {}, + "CongressionalDistrict": {}, + "Country": {}, + "County": {}, + "ElementarySchoolDistrict": {}, + "HighSchoolDistrict": {}, + "Neighborhood": {}, + "Place": {}, + "ProvisionalNode": {}, + "SchoolDistrict": {}, + "State": {}, + "Town": {}, + "Village": {} + }, + "observation_count": 2701901, + "time_series_count": 219993 + } + ], + "observation_count": 2701901, + "time_series_count": 219993 } }, "Count_Household_HasComputer": { @@ -86,6 +187,26 @@ ], "observation_count": 2805865, "time_series_count": 511517 + }, + { + "import_name": "CensusACS5YearSurvey_AggCountry", + "series_summary": [ + { + "series_key": { + "measurement_method": "CensusACS5yrSurvey", + "is_dc_aggregate": true + }, + "earliest_date": "2017", + "latest_date": "2023", + "place_type_summary": { + "Country": {} + }, + "observation_count": 7, + "time_series_count": 1 + } + ], + "observation_count": 7, + "time_series_count": 1 } } } \ No newline at end of file diff --git a/internal/server/spanner/golden/query_builder/get_event_collection_date_flood.sql b/internal/server/spanner/golden/query_builder/get_event_collection_date_flood.sql new file mode 100644 index 000000000..98f75ae3e --- /dev/null +++ b/internal/server/spanner/golden/query_builder/get_event_collection_date_flood.sql @@ -0,0 +1,6 @@ + @{force_join_order=true} + GRAPH DCGraph MATCH (event:Node)-[:Edge {predicate: 'affectedPlace', object_id: 'country/GBR'}]->(), (event)-[:Edge {predicate: 'typeOf', object_id: 'FloodEvent'}]->(), (event)-[:Edge {predicate: 'startDate'}]->(dateNode:Node) + RETURN DISTINCT + SUBSTR(dateNode.value, 1, 7) AS month + ORDER BY + month \ No newline at end of file diff --git a/internal/server/spanner/golden/query_builder/get_event_collection_dcids.sql b/internal/server/spanner/golden/query_builder/get_event_collection_dcids.sql new file mode 100644 index 000000000..45d837562 --- /dev/null +++ b/internal/server/spanner/golden/query_builder/get_event_collection_dcids.sql @@ -0,0 +1,6 @@ + @{force_join_order=true} + GRAPH DCGraph MATCH (event:Node)-[:Edge {predicate: 'affectedPlace', object_id: 'country/LBR'}]->(), (event)-[:Edge {predicate: 'typeOf', object_id: 'FireEvent'}]->(), (event)-[:Edge {predicate: 'startDate'}]->(dateNode:Node) + WHERE + SUBSTR(dateNode.value, 1, 7) = '2020-10' + RETURN DISTINCT + event.subject_id AS dcid \ No newline at end of file diff --git a/internal/server/spanner/golden/query_builder/get_event_collection_dcids_flood.sql b/internal/server/spanner/golden/query_builder/get_event_collection_dcids_flood.sql new file mode 100644 index 000000000..2335743b5 --- /dev/null +++ b/internal/server/spanner/golden/query_builder/get_event_collection_dcids_flood.sql @@ -0,0 +1,6 @@ + @{force_join_order=true} + GRAPH DCGraph MATCH (event:Node)-[:Edge {predicate: 'affectedPlace', object_id: 'country/GBR'}]->(), (event)-[:Edge {predicate: 'typeOf', object_id: 'FloodEvent'}]->(), (event)-[:Edge {predicate: 'startDate'}]->(dateNode:Node) + WHERE + SUBSTR(dateNode.value, 1, 7) = '2025-01' + RETURN DISTINCT + event.subject_id AS dcid \ No newline at end of file diff --git a/internal/server/spanner/golden/query_builder_test.go b/internal/server/spanner/golden/query_builder_test.go index d733d151e..07f6e7d9c 100644 --- a/internal/server/spanner/golden/query_builder_test.go +++ b/internal/server/spanner/golden/query_builder_test.go @@ -149,6 +149,18 @@ func TestGetEventCollectionDateQuery(t *testing.T) { } } +func TestGetEventCollectionDcidsQuery(t *testing.T) { + t.Parallel() + + for _, c := range eventCollectionDcidsTestCases { + goldenFile := c.golden + ".sql" + + runQueryBuilderGoldenTest(t, goldenFile, func(ctx context.Context) (interface{}, error) { + return spanner.GetEventCollectionDcidsQuery(c.placeDcid, c.eventType, c.date), nil + }) + } +} + // runQueryBuilderGoldenTest is a helper function that performs the golden file validation. func runQueryBuilderGoldenTest(t *testing.T, goldenFile string, fn goldenTestFunc) { t.Helper() diff --git a/internal/server/spanner/golden/query_cases_test.go b/internal/server/spanner/golden/query_cases_test.go index 98b1b19ca..bc493ede3 100644 --- a/internal/server/spanner/golden/query_cases_test.go +++ b/internal/server/spanner/golden/query_cases_test.go @@ -431,4 +431,57 @@ var eventCollectionDateTestCases = []struct { eventType: "FireEvent", golden: "get_event_collection_date", }, + { + placeDcid: "country/GBR", + eventType: "FloodEvent", + golden: "get_event_collection_date_flood", + }, +} + +var eventCollectionDcidsTestCases = []struct { + placeDcid string + eventType string + date string + golden string +}{ + { + placeDcid: "country/LBR", + eventType: "FireEvent", + date: "2020-10", + golden: "get_event_collection_dcids", + }, + { + placeDcid: "country/GBR", + eventType: "FloodEvent", + date: "2025-01", + golden: "get_event_collection_dcids_flood", + }, +} + +var eventCollectionTestCases = []struct { + placeDcid string + eventType string + date string + filterProp string + filterUnit string + filterLowerLimit float64 + filterUpperLimit float64 + golden string +}{ + { + placeDcid: "country/LBR", + eventType: "FireEvent", + date: "2020-10", + golden: "get_event_collection", + }, + { + placeDcid: "country/LBR", + eventType: "FireEvent", + date: "2020-10", + filterProp: "area", + filterUnit: "SquareKilometer", + filterLowerLimit: 100, + filterUpperLimit: 200, + golden: "get_event_collection_filtered", + }, } diff --git a/internal/server/spanner/golden/query_test.go b/internal/server/spanner/golden/query_test.go index 94873f68a..3508ef765 100644 --- a/internal/server/spanner/golden/query_test.go +++ b/internal/server/spanner/golden/query_test.go @@ -21,6 +21,7 @@ import ( "sort" "testing" + pbv1 "github.com/datacommonsorg/mixer/internal/proto/v1" "github.com/datacommonsorg/mixer/internal/server/datasources" "github.com/datacommonsorg/mixer/internal/server/spanner" "github.com/datacommonsorg/mixer/test" @@ -224,6 +225,59 @@ func TestGetEventCollectionDate(t *testing.T) { } } +func TestFindEventDcids(t *testing.T) { + client := test.NewSpannerClient() + if client == nil { + return + } + + t.Parallel() + + for _, c := range eventCollectionDcidsTestCases { + goldenFile := c.golden + ".json" + + runQueryGoldenTest(t, goldenFile, func(ctx context.Context) (interface{}, error) { + return client.GetEventCollectionDcids(ctx, c.placeDcid, c.eventType, c.date) + }) + } +} + +func TestGetEventCollection(t *testing.T) { + client := test.NewSpannerClient() + if client == nil { + return + } + + t.Parallel() + + for _, c := range eventCollectionTestCases { + goldenFile := c.golden + ".json" + + runQueryGoldenTest(t, goldenFile, func(ctx context.Context) (interface{}, error) { + req := &pbv1.EventCollectionRequest{ + AffectedPlaceDcid: c.placeDcid, + EventType: c.eventType, + Date: c.date, + } + if c.filterProp != "" { + req.FilterProp = c.filterProp + req.FilterUnit = c.filterUnit + req.FilterUpperLimit = c.filterUpperLimit + req.FilterLowerLimit = c.filterLowerLimit + } + res, err := client.GetEventCollection(ctx, req) + if err != nil { + return nil, err + } + // Trim to 10 events to avoid very large golden files. + if len(res.Events) > 10 { + res.Events = res.Events[:10] + } + return res, nil + }) + } +} + // runQueryGoldenTest is a helper function that performs the golden file validation. func runQueryGoldenTest(t *testing.T, goldenFile string, fn goldenTestFunc) { t.Helper() diff --git a/internal/server/spanner/query.go b/internal/server/spanner/query.go index cc1be5455..f453de2cc 100644 --- a/internal/server/spanner/query.go +++ b/internal/server/spanner/query.go @@ -19,13 +19,20 @@ import ( "context" "fmt" "log/slog" + "net/url" + "sort" + "strconv" + "strings" "time" "cloud.google.com/go/spanner" "github.com/datacommonsorg/mixer/internal/metrics" pb "github.com/datacommonsorg/mixer/internal/proto" + pbv1 "github.com/datacommonsorg/mixer/internal/proto/v1" + "github.com/datacommonsorg/mixer/internal/server/datasources" v2 "github.com/datacommonsorg/mixer/internal/server/v2" "github.com/datacommonsorg/mixer/internal/translator/types" + "github.com/datacommonsorg/mixer/internal/util" "google.golang.org/api/iterator" "google.golang.org/grpc/codes" "google.golang.org/protobuf/encoding/protojson" @@ -37,6 +44,17 @@ const ( maxHops = 10 where = "\n\t\tWHERE\n\t\t\t" and = "\n\t\t\tAND " + + // Special edge predicates. + predAffectedPlace = "affectedPlace" + predStartDate = "startDate" + predStartLocation = "startLocation" + predProvenance = "provenance" + predTypeOf = "typeOf" + predGeoJsonCoordinates = "geoJsonCoordinates" + predName = "name" + predUrl = "url" + predDomain = "domain" ) // GetNodeProps retrieves node properties from Spanner given a list of IDs and a direction and returns a map. @@ -231,6 +249,262 @@ func (sc *spannerDatabaseClient) GetEventCollectionDate(ctx context.Context, pla return res, nil } +// GetEventCollection retrieves and filters event collection from Spanner. +func (sc *spannerDatabaseClient) GetEventCollection(ctx context.Context, req *pbv1.EventCollectionRequest) (*pbv1.EventCollection, error) { + // Get event DCIDs + dcids, err := sc.GetEventCollectionDcids(ctx, req.AffectedPlaceDcid, req.EventType, req.Date) + if err != nil { + return nil, fmt.Errorf("failed to get event dcids: %w", err) + } + if len(dcids) == 0 { + return &pbv1.EventCollection{}, nil + } + + // Get properties for all DCIDs + arc := &v2.Arc{ + Out: true, + SingleProp: "*", + } + edgesMap, err := sc.GetNodeEdgesByID(ctx, dcids, arc, datasources.DefaultPageSize, 0) + if err != nil { + return nil, fmt.Errorf("failed to get node edges: %w", err) + } + + // Filter and Assemble + res := assembleEventCollection(edgesMap, req) + + // Fetch and populate provenance info. + if err := sc.populateProvenanceInfo(ctx, res); err != nil { + return nil, err + } + + return res, nil +} +func (sc *spannerDatabaseClient) populateProvenanceInfo(ctx context.Context, res *pbv1.EventCollection) error { + provDcids := []string{} + seen := map[string]bool{} + for _, event := range res.Events { + if event.ProvenanceId != "" && !seen[event.ProvenanceId] { + seen[event.ProvenanceId] = true + provDcids = append(provDcids, event.ProvenanceId) + } + } + + if len(provDcids) == 0 { + return nil + } + + provArc := &v2.Arc{ + Out: true, + BracketProps: []string{predUrl, predName, predDomain}, + } + provEdgesMap, err := sc.GetNodeEdgesByID(ctx, provDcids, provArc, datasources.DefaultPageSize, 0) + if err != nil { + return fmt.Errorf("failed to get provenance info: %w", err) + } + + for provDcid, edges := range provEdgesMap { + info := &pbv1.EventCollection_ProvenanceInfo{} + res.ProvenanceInfo[provDcid] = info + for _, edge := range edges { + switch edge.Predicate { + case predUrl: + info.ProvenanceUrl = edge.Value + case predName: + info.ImportName = edge.Value + case predDomain: + info.Domain = edge.Value + } + } + // Fallback if domain still empty. + if info.Domain == "" && info.ProvenanceUrl != "" { + info.Domain = parseDomain(info.ProvenanceUrl) + } + } + + return nil +} + +// parseDomain parses the URL to get the host domain. +func parseDomain(provUrl string) string { + u, err := url.Parse(provUrl) + if err != nil { + return "" + } + host := u.Hostname() + parts := strings.Split(host, ".") + if len(parts) >= 2 { + return strings.Join(parts[len(parts)-2:], ".") + } + return host +} + + +func assembleEventCollection(edgesMap map[string][]*Edge, req *pbv1.EventCollectionRequest) *pbv1.EventCollection { + res := &pbv1.EventCollection{ + Events: []*pbv1.EventCollection_Event{}, + ProvenanceInfo: make(map[string]*pbv1.EventCollection_ProvenanceInfo), + } + + for dcid, edges := range edgesMap { + event := assembleAndFilterEvent(dcid, edges, req) + if event != nil { + res.Events = append(res.Events, event) + } + } + + // Sort events by DCID for determinism. + sort.Slice(res.Events, func(i, j int) bool { + return res.Events[i].Dcid < res.Events[j].Dcid + }) + + return res +} + +// assembleAndFilterEvent assembles an event from its edges and filters it based on the request. +// Returns nil if the event should be filtered out. +func assembleAndFilterEvent(dcid string, edges []*Edge, req *pbv1.EventCollectionRequest) *pbv1.EventCollection_Event { + event := &pbv1.EventCollection_Event{ + Dcid: dcid, + Places: []string{}, + Dates: []string{}, + GeoLocations: []*pbv1.EventCollection_GeoLocation{}, // Initialize + PropVals: make(map[string]*pbv1.EventCollection_ValList), + } + + for _, edge := range edges { + populateSpecialFields(event, edge) + populatePropVals(event, edge) + } + + // Filter events. + // We must filter AFTER populating because we need the full event data + // (e.g. PropVals) for filtering logic (keepEvent). + if !keepEvent(event, req) { + return nil + } + + cleanUpPropVals(event) + + return event +} + +func populateSpecialFields(event *pbv1.EventCollection_Event, edge *Edge) { + switch edge.Predicate { + case predAffectedPlace: + // Exclude S2Cell places as per proto contract (proto/v1/event.proto). + if !strings.HasPrefix(edge.Value, "s2CellId/") { + event.Places = append(event.Places, edge.Value) + } + case predStartDate: + // Do NOT trim date. + event.Dates = append(event.Dates, edge.Value) + case predStartLocation: + // Populate GeoLocations from startLocation value. + populateGeoLocation(event, edge.Value) + } +} + +func populateGeoLocation(event *pbv1.EventCollection_Event, value string) { + // Note: The startLocation value in Spanner is usually a latLong/ DCID (e.g. latLong/577521_-958960). + // We parse it here for performance to avoid an extra database roundtrip. + // + // TODO(task): Revisit this optimization if we encounter valid startLocation values + // that are NOT latLong/ DCIDs but still need to be resolved to points, or if the + // assumption that dcids always contain coordinates is not true. + if strings.HasPrefix(value, "latLong/") { + parts := strings.Split(strings.TrimPrefix(value, "latLong/"), "_") + if len(parts) == 2 { + lat, err1 := strconv.ParseFloat(parts[0], 64) + lon, err2 := strconv.ParseFloat(parts[1], 64) + if err1 == nil && err2 == nil { + event.GeoLocations = append(event.GeoLocations, &pbv1.EventCollection_GeoLocation{ + Geo: &pbv1.EventCollection_GeoLocation_Point_{ + Point: &pbv1.EventCollection_GeoLocation_Point{ + Latitude: proto.Float64(lat / 100000.0), + Longitude: proto.Float64(lon / 100000.0), + }, + }, + }) + return // Success + } + } + } + + slog.Warn("startLocation is not a valid latLong/ DCID, skipping parsing optimization", "value", value) +} + +func populatePropVals(event *pbv1.EventCollection_Event, edge *Edge) { + val := edge.Value + if edge.Predicate == predGeoJsonCoordinates && len(edge.Bytes) > 0 { + if len(edge.Bytes) > 2 && edge.Bytes[0] == 0x1f && edge.Bytes[1] == 0x8b { + decompressed, err := util.Unzip(edge.Bytes) + if err == nil { + val = string(decompressed) + } else { + slog.Error("failed to decompress geoJsonCoordinates", "err", err, "dcid", event.Dcid) + val = "" + } + } else { + val = string(edge.Bytes) + } + } + if _, ok := event.PropVals[edge.Predicate]; !ok { + event.PropVals[edge.Predicate] = &pbv1.EventCollection_ValList{Vals: []string{}} + } + event.PropVals[edge.Predicate].Vals = append(event.PropVals[edge.Predicate].Vals, val) + + if edge.Provenance != "" { + event.ProvenanceId = edge.Provenance + } +} + +func cleanUpPropVals(event *pbv1.EventCollection_Event) { + // Clean up PropVals (remove specialized fields to match V2). + delete(event.PropVals, predAffectedPlace) + delete(event.PropVals, predStartDate) + delete(event.PropVals, predStartLocation) + delete(event.PropVals, predProvenance) + delete(event.PropVals, predTypeOf) +} + +func keepEvent(event *pbv1.EventCollection_Event, req *pbv1.EventCollectionRequest) bool { + if req.FilterProp == "" { + return true + } + for prop, vals := range event.GetPropVals() { + if prop == req.FilterProp { + if len(vals.Vals) == 0 { + return false + } + valStr := strings.TrimSpace(strings.TrimPrefix(vals.Vals[0], req.FilterUnit)) + v, err := strconv.ParseFloat(valStr, 64) + if err != nil { + return false + } + return v >= req.FilterLowerLimit && v <= req.FilterUpperLimit + } + } + return false +} + +// GetEventCollectionDcids retrieves event DCIDs from Spanner. +func (sc *spannerDatabaseClient) GetEventCollectionDcids(ctx context.Context, placeID, eventType, date string) ([]string, error) { + stmt := GetEventCollectionDcidsQuery(placeID, eventType, date) + rows, err := queryDynamic(ctx, sc, *stmt) + if err != nil { + return nil, err + } + + var dcids []string + for _, row := range rows { + if len(row) > 0 { + dcids = append(dcids, row[0]) + } + } + return dcids, nil +} + func (sc *spannerDatabaseClient) Sparql(ctx context.Context, nodes []types.Node, queries []*types.Query, opts *types.QueryOptions) ([][]string, error) { query, err := SparqlQuery(nodes, queries, opts) if err != nil { diff --git a/internal/server/spanner/query_builder.go b/internal/server/spanner/query_builder.go index 656f42740..127f46401 100644 --- a/internal/server/spanner/query_builder.go +++ b/internal/server/spanner/query_builder.go @@ -426,3 +426,14 @@ func GetEventCollectionDateQuery(placeID, eventType string) *spanner.Statement { }, } } + +func GetEventCollectionDcidsQuery(placeID, eventType, date string) *spanner.Statement { + return &spanner.Statement{ + SQL: statements.getEventCollectionDcids, + Params: map[string]interface{}{ + "placeID": placeID, + "eventType": eventType, + "date": date, + }, + } +} diff --git a/internal/server/spanner/statements.go b/internal/server/spanner/statements.go index a42c314f5..37de957af 100644 --- a/internal/server/spanner/statements.go +++ b/internal/server/spanner/statements.go @@ -89,6 +89,8 @@ var statements = struct { getCacheData string // Fetch event dates for a given type and location. getEventCollectionDate string + // Fetch events for a given type, location and date. + getEventCollectionDcids string }{ getCompletionTimestamp: ` SELECT CompletionTimestamp @@ -311,4 +313,10 @@ var statements = struct { SUBSTR(dateNode.value, 1, 7) AS month ORDER BY month`, + getEventCollectionDcids: ` @{force_join_order=true} + GRAPH DCGraph MATCH (event:Node)-[:Edge {predicate: 'affectedPlace', object_id: @placeID}]->(), (event)-[:Edge {predicate: 'typeOf', object_id: @eventType}]->(), (event)-[:Edge {predicate: 'startDate'}]->(dateNode:Node) + WHERE + SUBSTR(dateNode.value, 1, 7) = @date + RETURN DISTINCT + event.subject_id AS dcid`, }