Skip to content

Commit e325183

Browse files
jo-chemlajo-chemla
authored
Merge pull request #30 from Iconem/UP42_APOLLO_fixes
fix: handle new UP42 and Apollo apis #17
2 parents bb5d786 + 527f162 commit e325183

12 files changed

+691
-506
lines changed

‎package.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@
4747
"crypto-browserify": "^3.12.0",
4848
"date-fns": "^2.29.3",
4949
"esbuild": "0.17.11",
50+
"fast-deep-equal": "^3.1.3",
5051
"ky": "^0.33.2",
5152
"mapbox-gl": "^2.13.0",
5253
"mapbox-gl-draw-rectangle-mode": "^1.0.4",

‎src/archive-apis/search-apollo.tsx‎

Lines changed: 69 additions & 75 deletions
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,30 @@
11
// Code for searching APOLLO MAPPING API
22

33
import ky from 'ky'
4-
import { Providers, maxAbs, filterFeaturesWithSearchParams } from './search-utilities'
4+
import { Providers, maxAbs, filterFeaturesWithSearchParams, processInChunks } from './search-utilities'
55
import { v4 as uuidv4 } from 'uuid'
66
import bboxPolygon from '@turf/bbox-polygon'
77
import { log } from '../utilities'
8-
import type GeoJSON from 'react-map-gl'
9-
10-
// https://imagehunter.apollomapping.com/
118

129
const APOLLO_API_URL = 'https://imagehunter-api.apollomapping.com'
1310
const APOLLO_DOMAIN = 'https://imagehunter.apollomapping.com'
1411

1512
const searchApollo = async (searchSettings, apolloApikey, searchPolygon: any | null = null, setters: any | null = null): Promise<any> => {
13+
14+
const generateXAuthToken = async (): Promise<string> => {
15+
const now = new Date();
16+
const day = now.getUTCDate();
17+
const hour = now.getUTCHours();
18+
const secret = "iGsZ1wMw8xERUZrxNPvBt2TlFTFcN3P2";
19+
const raw = `${day}${hour}${secret}`;
20+
const msgUint8 = new TextEncoder().encode(raw); // encode string to Uint8Array
21+
const hashBuffer = await window.crypto.subtle.digest("SHA-256", msgUint8); // hash
22+
const hashArray = Array.from(new Uint8Array(hashBuffer)); // buffer -> byte array
23+
const hashHex = hashArray.map(b => b.toString(16).padStart(2, "0")).join(""); // byte array -> hex string
24+
25+
return hashHex;
26+
}
27+
1628
const apolloPayload = {
1729
startDate: searchSettings.startDate.toISOString().substring(0, 19), // '1970-01-01T12:00:00',
1830
endDate: searchSettings.endDate.toISOString().substring(0, 19),
@@ -29,32 +41,17 @@ const searchApollo = async (searchSettings, apolloApikey, searchPolygon: any | n
2941
stereo: false,
3042
seasonal: false,
3143
}
44+
3245
const apolloPayloadBody = new URLSearchParams(apolloPayload as any).toString()
3346
// Better have a look at maxar payload construction
3447

3548
// WIP, returns 401 unauthorized because origin and referer headers are overwritten on post
36-
// return {
37-
// searchResultsJson: {
38-
// 'features': [],
39-
// type: 'FeatureCollection'
40-
// }
41-
// }
4249

4350
// Returns 401 Unauthorized probably because request is actually sent to http non-sll server url because dev server has no tls and origin and referer are overwritten by browser to localhost
4451
// Tried using a CORS proxy, with no luck like [cors-anywhere](https://github.com/Rob--W/cors-anywhere/#documentation) public one or [allorigins](https://allorigins.win/) or any recent one listed [here](https://nordicapis.com/10-free-to-use-cors-proxies/)
4552

46-
// const requestOptions = {headers: { 'Host': APOLLO_DOMAIN, 'Origin': APOLLO_DOMAIN, 'Referer': APOLLO_DOMAIN, }, method: 'POST'};
47-
// let apolloRequest = new Request(APOLLO_SEARCH_URL, requestOptions);
4853
const apolloRequest = `${APOLLO_API_URL}/ajax/search`
49-
// let apolloRequest = 'https://api.allorigins.win/get?url=https%3A%2F%2Freqres.in%2Fapi%2Fusers'
50-
// let apolloRequest = 'https://test.cors.workers.dev/?https%3A%2F%2Freqres.in%2Fapi%2Fusers'
51-
// apolloRequest = 'http://localhost:8010/proxy/ajax/search'
52-
// let apolloRequest = `https://api.allorigins.win/post?url=${encodeURIComponent(APOLLO_SEARCH_URL)}`
53-
// const testPayload = {
54-
// name: 'morpheudezetterz',
55-
// job: 'letrezterzder',
56-
// }
57-
54+
const token = await generateXAuthToken()
5855
const apolloResultsRaw = await ky
5956
.post(apolloRequest, {
6057
headers: {
@@ -63,9 +60,9 @@ const searchApollo = async (searchSettings, apolloApikey, searchPolygon: any | n
6360
Referer: APOLLO_DOMAIN,
6461
'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
6562
'Content-Length': `${apolloPayloadBody.length}`,
66-
// 'Content-Length': '867'
63+
"X-Auth-Token": token,
6764
},
68-
body: apolloPayloadBody, // testPayload, apolloPayload, fakePayload
65+
body: apolloPayloadBody,
6966
})
7067
.json()
7168

@@ -79,7 +76,7 @@ const searchApollo = async (searchSettings, apolloApikey, searchPolygon: any | n
7976
// A better way could be to add a callback on hover in datagrid if preview does not exist to retrieve its url
8077
// Initiate search for previews/thumbnails
8178
// TODO REDO
82-
void getApolloPreviewsAsync(searchResultsJson).then((results) => {
79+
void getApolloPreviewsAsync(searchResultsJson, token).then((results) => {
8380
if (setters) {
8481
// const searchResults = {
8582
// input: searchPolygon,
@@ -131,58 +128,55 @@ const formatApolloResults = (apolloResultsRaw): GeoJSON.FeatureCollection => {
131128
}
132129

133130
const timer = async (ms): Promise<any> => await new Promise((resolve) => setTimeout(resolve, ms))
134-
135-
const getApolloPreviewsAsync = async (apolloResults): Promise<void> => {
136-
const chunkSize = 10
137-
// Send batches of chunkSize POST requests to ApolloMapping API server
138-
for (let i = 0; i < apolloResults.features.length; i += chunkSize) {
139-
const chunk = apolloResults.features.slice(i, i + chunkSize)
140-
const errors: string[] = []
141-
// Await all promises of chunk fetching
142-
await Promise.all(
143-
chunk.map(async (feature) => {
144-
const previewPayload = {
145-
catid: feature.properties.id,
146-
satellite: 'scene',
147-
satelliteShortName: feature.properties.providerProperties.satelliteShortName,
148-
isRefresh: false,
149-
forceHighestQuality: false, // Todo: test true
131+
const getApolloPreviewsAsync = async (apolloResults, token): Promise<void> => {
132+
const errors: string[] = [];
133+
134+
await processInChunks(
135+
async (feature: any) => {
136+
const previewPayload = {
137+
catid: feature.properties.id,
138+
satellite: 'scene',
139+
satelliteShortName: feature.properties.providerProperties.satelliteShortName,
140+
isRefresh: false,
141+
forceHighestQuality: false,
142+
};
143+
144+
const previewPayloadBody = new URLSearchParams(previewPayload as any).toString();
145+
146+
await ky
147+
.post(`${APOLLO_API_URL}/ajax/get_preview_image`, {
148+
headers: {
149+
Host: APOLLO_DOMAIN,
150+
Origin: APOLLO_DOMAIN,
151+
Referer: APOLLO_DOMAIN,
152+
'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
153+
'Content-Length': `${previewPayloadBody.length}`,
154+
"x-auth-token": token,
155+
},
156+
body: previewPayloadBody,
157+
timeout: 30000,
158+
})
159+
.json()
160+
.then((previewUrlObject) => {
161+
feature.properties.thumbnail_uri = `${APOLLO_API_URL}${(previewUrlObject as any).path}`;
162+
feature.properties.preview_uri = feature.properties.thumbnail_uri;
163+
})
164+
.catch(() => {
165+
errors.push('Error during ky post request to get Apollo preview image');
166+
});
167+
},
168+
{
169+
items: apolloResults.features,
170+
chunkSize: 10,
171+
delayBetweenChunks: 500,
172+
usePromiseAllSettled: false, // Stop on error like original
173+
onChunkComplete: (chunk, chunkIndex, totalChunks) => {
174+
if (errors.length > 0) {
175+
throw new Error(`ApolloMapping Previews fetch - Error during chunk ${chunkIndex}`);
150176
}
151-
const previewPayloadBody = new URLSearchParams(previewPayload as any).toString()
152-
// console.log('Fetching preview via ky.post for ', feature.properties.id)
153-
// console.log('REQUESTING APOLLO PREVIEW', previewPayload.catid, previewPayload, previewUrlObject, feature.properties.thumbnail_uri)
154-
await ky
155-
.post(`${APOLLO_API_URL}/ajax/get_preview_image`, {
156-
headers: {
157-
Host: APOLLO_DOMAIN,
158-
Origin: APOLLO_DOMAIN,
159-
Referer: APOLLO_DOMAIN,
160-
'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
161-
'Content-Length': `${previewPayloadBody.length}`,
162-
},
163-
body: previewPayloadBody,
164-
timeout: 30000,
165-
})
166-
.json()
167-
.then((previewUrlObject) => {
168-
// console.log('ApolloMapping, found preview ', previewUrlObject)
169-
feature.properties.thumbnail_uri = `${APOLLO_API_URL}${(previewUrlObject as any).path as string}`
170-
feature.properties.preview_uri = feature.properties.thumbnail_uri
171-
})
172-
.catch(() => {
173-
errors.push('Error during ky post request to get Apollo preview image')
174-
})
175-
})
176-
)
177-
178-
if (errors.length > 0) {
179-
log(`ApolloMapping Previews fetch - Catched error during chunk previews fetch for loop ${i}, so break master loop by return`)
180-
throw new Error(`ApolloMapping Previews fetch - Catched error during chunk previews fetch for loop ${i}, so break master loop by return`)
181-
// throw new Error(errors.join());
182-
// return
177+
}
183178
}
184-
await timer(3000)
185-
}
186-
}
179+
);
180+
};
187181

188182
export default searchApollo

‎src/archive-apis/search-skyfi.tsx‎

Lines changed: 39 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -14,11 +14,13 @@ import { parse as wkt_parse, stringify as wkt_stringify } from 'wellknown'
1414
// App: https://app.skyfi.com/explore
1515
// POST on https://app.skyfi.com/api/archive-available
1616
// Payload: {"clientType":"DESKTOP","fromDate":"2018-12-31T23:00:00.000Z","toDate":"2023-01-24T10:21:44.465Z","maxCloudCoveragePercent":20,"resolutions":["VERY HIGH","HIGH"],"sensors":["DAY","NIGHT","MULTISPECTRAL"],"imageCropping":{"wktString":"POLYGON((-77.05679665567264 38.899388312735795,-77.02040942302703 38.899388312735795,-77.02040942302703 38.92503663542644,-77.05679665567264 38.92503663542644,-77.05679665567264 38.899388312735795))"},"page":0,"pageSize":25}
17+
console.log('start');
1718

18-
const SKYFI_SEARCH_URL = 'https://app.skyfi.com/api/archive-available'
19+
// const SKYFI_SEARCH_URL = 'https://app.skyfi.com/api/archive-available'
20+
const SKYFI_SEARCH_URL = 'https://app.skyfi.com/platform-api/archives'
1921
const pageSize = 25
2022
const lookForNextPage = true
21-
const skyfiApiKey = 'eh6qPPge7f88EJPp'
23+
const skyfiApiKey = process.env.SKYFI_API_KEY
2224

2325
// Not useful at the moment
2426
// const getSkyfiBearer = (apikey): string => {
@@ -59,31 +61,49 @@ const searchSkyfi = async (searchSettings, skyfiApikey, searchPolygon = null, se
5961
toDate: searchSettings.endDate.toISOString(), // "2023-01-24T10:21:44.465Z",
6062
maxCloudCoveragePercent: searchSettings.cloudCoverage,
6163
resolutions: resolutionArray,
62-
clientType: 'DESKTOP',
63-
sensors: ['DAY', 'NIGHT', 'MULTISPECTRAL'],
64-
imageCropping: {
65-
wktString: coordinatesWkt,
66-
},
67-
page: pageIdx,
68-
pageSize,
64+
// isOpendata: false,
65+
// bandsCount: [
66+
// 1,
67+
// 3,
68+
// 4
69+
// ],
70+
openData: false,
71+
// clientType: 'DESKTOP',
72+
// sensors: ['DAY', 'NIGHT', 'MULTISPECTRAL'],
73+
// sensors: [ "DAY", "SAR", "STEREO"],
74+
providers: [
75+
"PLANET",
76+
"GEOSAT",
77+
"SATELLOGIC",
78+
"SIWEI",
79+
"IMPRO",
80+
"URBAN_SKY",
81+
"VEXCEL",
82+
"NSL",
83+
"UMBRA"
84+
],
85+
// imageCropping: {
86+
// wktString: coordinatesWkt,
87+
// },
88+
aoi: coordinatesWkt,
89+
// page: pageIdx,
90+
pageSize: pageSize,
6991
}
7092
log('skyfi PAYLOAD: \n', skyfiPayload)
7193

7294
const contentLength = `${JSON.stringify(skyfiPayload).length}`
73-
console.log('contentLength', contentLength, skyfiPayload, JSON.stringify(skyfiPayload), JSON.stringify(skyfiPayload).length)
7495
const skyfiResultsRaw = await ky
7596
.post(SKYFI_SEARCH_URL, {
7697
headers: {
7798
// 'skyfi-api-key': skyfiApiKey,
78-
'skyfi-api-key': ' eh6qPPge7f88EJPp',
79-
'content-length': contentLength,
80-
81-
origin: ' https://app.skyfi.com',
82-
referer: ' https://app.skyfi.com/explore/',
83-
accept: ' application/json',
84-
'cache-control': ' no-cache',
85-
'content-type': ' application/json',
86-
'skyfi-client-agent': ' DESKTOP-1.8.0',
99+
'X-Skyfi-Api-Key': skyfiApiKey,
100+
// 'content-length': contentLength,
101+
// Origin: 'https://app.skyfi.com',
102+
// Referer: 'https://app.skyfi.com/explore/',
103+
Accept: 'application/json',
104+
'content-type': 'application/json',
105+
// 'cache-control': ' no-cache',
106+
// 'skyfi-client-agent': ' DESKTOP-1.8.0',
87107
},
88108
json: skyfiPayload,
89109
})

‎src/archive-apis/search-skywatch.tsx‎

Lines changed: 17 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -14,11 +14,11 @@ const SKYWATCH_SEARCH_URL_JWT = 'https://api.skywatch.co/auth0-jwt/earthcache/ar
1414
const performSkywatchSearch = async (skywatchApikey, searchId): Promise<any> =>
1515
skywatchApikey?.includes('Bearer')
1616
? await ky.get(`${SKYWATCH_SEARCH_URL_JWT}/${searchId.data.id as string}/search_results`, {
17-
headers: { authorization: skywatchApikey },
18-
})
17+
headers: { authorization: skywatchApikey },
18+
})
1919
: await ky.get(`${SKYWATCH_SEARCH_URL}/${searchId.data.id as string}/search_results`, {
20-
headers: { 'x-api-key': skywatchApikey },
21-
})
20+
headers: { 'x-api-key': skywatchApikey },
21+
})
2222

2323
const MAX_QUERY_COUNT = 8
2424

@@ -50,20 +50,20 @@ const searchSkywatch = async (searchSettings, skywatchApikey, searchPolygon = nu
5050

5151
const searchId = skywatchApikey?.includes('Bearer')
5252
? await ky
53-
.post(SKYWATCH_SEARCH_URL_JWT, {
54-
headers: {
55-
authorization: skywatchApikey,
56-
'content-length': `${JSON.stringify(skywatchPayload).length}`,
57-
},
58-
json: skywatchPayload,
59-
})
60-
.json()
53+
.post(SKYWATCH_SEARCH_URL_JWT, {
54+
headers: {
55+
authorization: skywatchApikey,
56+
'content-length': `${JSON.stringify(skywatchPayload).length}`,
57+
},
58+
json: skywatchPayload,
59+
})
60+
.json()
6161
: await ky
62-
.post(SKYWATCH_SEARCH_URL, {
63-
headers: { 'x-api-key': skywatchApikey },
64-
json: skywatchPayload,
65-
})
66-
.json()
62+
.post(SKYWATCH_SEARCH_URL, {
63+
headers: { 'x-api-key': skywatchApikey },
64+
json: skywatchPayload,
65+
})
66+
.json()
6767
log('SKYWATCH searchId: \n', searchId, '\n')
6868

6969
let searchResultsRaw

0 commit comments

Comments
 (0)