Skip to content

Commit 9c7aa4f

Browse files
btrybtry
committed
feat(DataSource\CarbonIntensity): response caching
1 parent 9379517 commit 9c7aa4f

File tree

9 files changed

+264
-83
lines changed

9 files changed

+264
-83
lines changed

‎src/CarbonIntensity.php‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -213,7 +213,7 @@ public function getFirstKnownDate(string $zone_name, string $source_name): ?Date
213213
*/
214214
public function downloadOneZone(ClientInterface $data_source, string $zone_name, int $limit = 0, ?ProgressBar $progress_bar = null): int
215215
{
216-
$start_date = $this->getDownloadStartDate($zone_name, $data_source);
216+
$start_date = $this->getDownloadStartDate();
217217

218218
$total_count = 0;
219219

@@ -279,11 +279,9 @@ public function downloadOneZone(ClientInterface $data_source, string $zone_name,
279279
/**
280280
* Get the oldest date where data are required
281281
*
282-
* @param string $zone_name ignored for now; zone to examine
283-
* @param ClientInterface $data_source ignored for now; data source
284282
* @return DateTimeImmutable|null
285283
*/
286-
public function getDownloadStartDate(string $zone_name, ClientInterface $data_source): ?DateTimeImmutable
284+
public function getDownloadStartDate(): ?DateTimeImmutable
287285
{
288286
// Get the default oldest date od data to download
289287
$start_date = new DateTime(self::MIN_HISTORY_LENGTH);

‎src/Command/CollectCarbonIntensityCommand.php‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -169,7 +169,7 @@ protected function execute(InputInterface $input, OutputInterface $output)
169169
$carbon_intensity->downloadOneZone($this->client, $this->zones[$zone_code], 0, new ProgressBar($this->output));
170170

171171
// Find start and stop dates to cover
172-
$start_date = $carbon_intensity->getDownloadStartDate($this->zones[$zone_code], $this->client);
172+
$start_date = $carbon_intensity->getDownloadStartDate();
173173
$gaps = $carbon_intensity->findGaps($this->source_id, $zone->getID(), $start_date);
174174

175175
// Count the hours not covered by any sample

‎src/DataSource/CarbonIntensity/AbstractClient.php‎

Lines changed: 21 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -157,8 +157,17 @@ public function getZones(array $crit = []): array
157157

158158
public function fullDownload(string $zone, DateTimeImmutable $start_date, DateTimeImmutable $stop_date, CarbonIntensity $intensity, int $limit = 0, ?ProgressBar $progress_bar = null): int
159159
{
160+
if ($start_date >= $stop_date) {
161+
return 0;
162+
}
163+
// Round start date to beginning of month
164+
$start_date = $start_date->setTime(0, 0, 0, 0)->setDate((int) $start_date->format('Y'), (int) $start_date->format('m'), 1);
165+
$stop_date = $stop_date->setTime(0, 0, 0, 0)->setDate((int) $stop_date->format('Y'), (int) $stop_date->format('m'), 1);
160166
$count = 0;
161167
$saved = 0;
168+
if ($start_date == $stop_date){
169+
$stop_date = $stop_date->add(new DateInterval('P1M'));
170+
}
162171

163172
/**
164173
* Huge quantity of SQL queries will be executed
@@ -171,9 +180,18 @@ public function fullDownload(string $zone, DateTimeImmutable $start_date, DateTi
171180
// enpty string in PHPUnit environment
172181
$memory_limit = null;
173182
}
174-
foreach ($this->sliceDateRangeByMonth($start_date, $stop_date) as $slice) {
183+
184+
// Traverse each month from start_date to end_date
185+
$current_date = DateTime::createFromImmutable($start_date);
186+
while ($current_date < $stop_date) {
187+
$next_month = clone $current_date;
188+
$next_month->add(new DateInterval('P1M'));
175189
try {
176-
$data = $this->fetchRange($slice['start'], $slice['stop'], $zone);
190+
$data = $this->fetchRange(
191+
DateTimeImmutable::createFromMutable($current_date),
192+
DateTimeImmutable::createFromMutable($next_month),
193+
$zone
194+
);
177195
} catch (AbortException $e) {
178196
break;
179197
}
@@ -194,8 +212,8 @@ public function fullDownload(string $zone, DateTimeImmutable $start_date, DateTi
194212
// 8 MB memory left, emergency exit
195213
return $saved > 0 ? $count : -$count;
196214
}
215+
$current_date = $next_month;
197216
}
198-
199217
return $saved > 0 ? $count : -$count;
200218
}
201219

‎src/DataSource/CarbonIntensity/ElectricityMaps/Client.php‎

Lines changed: 45 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,6 @@
4141
use GlpiPlugin\Carbon\Source;
4242
use GlpiPlugin\Carbon\Zone;
4343
use GlpiPlugin\Carbon\Source_Zone;
44-
use Config as GlpiConfig;
4544
use GLPIKey;
4645
use GlpiPlugin\Carbon\DataSource\CarbonIntensity\AbortException;
4746
use GlpiPlugin\Carbon\DataSource\CarbonIntensity\AbstractClient;
@@ -264,16 +263,45 @@ public function fetchDay(DateTimeImmutable $day, string $zone): array
264263

265264
public function fetchRange(DateTimeImmutable $start, DateTimeImmutable $stop, string $zone): array
266265
{
267-
268266
// TODO: get zones from GLPI locations
269267
$params = [
270268
'zone' => $zone,
271269
];
272270

271+
$base_path = GLPI_PLUGIN_DOC_DIR . '/carbon/carbon_intensity/' . $this->getSourceName() . '/' . $zone;
272+
$cache_file = $this->getCacheFilename(
273+
$base_path,
274+
$start,
275+
$stop
276+
);
277+
// If cached file exists, use it
278+
if (file_exists($cache_file)) {
279+
$full_response = json_decode(file_get_contents($cache_file), true);
280+
return $full_response;
281+
}
282+
@mkdir(dirname($cache_file), 0755, true);
283+
284+
// Set timezone to +00:00 and extend range by 12 hours on each side
285+
$request_start = $start->setTimezone(new DateTimeZone('+0000'))->sub(new DateInterval('PT12H'));
286+
$request_stop = $stop->setTimezone(new DateTimeZone('+0000'))->add(new DateInterval('PT12H'));
273287
$this->step = 60;
274288

275-
$response = $this->client->request('GET', $this->base_url . self::PAST_URL, ['query' => $params]);
276-
if (!$response) {
289+
$step = new DateInterval('P10D');
290+
$full_response = [];
291+
$current_date = DateTime::createFromImmutable($request_start);
292+
while ($current_date < $request_stop) {
293+
$response = $this->client->request('GET', $this->base_url . self::PAST_URL, ['query' => $params]);
294+
if (!$full_response) {
295+
$full_response = $response;
296+
} else {
297+
$full_response['data'] = array_merge($full_response['data'], $response['data']);
298+
}
299+
$current_date->add($step);
300+
if ($current_date > $request_stop) {
301+
$current_date = $request_stop;
302+
}
303+
}
304+
if (!$full_response) {
277305
return [];
278306
}
279307

@@ -286,7 +314,9 @@ public function fetchRange(DateTimeImmutable $start, DateTimeImmutable $stop, st
286314
return [];
287315
}
288316

289-
return $$response['history'];
317+
$json = json_encode($full_response);
318+
file_put_contents($cache_file, $json);
319+
return $full_response['data'];
290320
}
291321

292322
protected function formatOutput(array $response, int $step): array
@@ -369,4 +399,14 @@ protected function sliceDateRangeByDay(DateTimeImmutable $start, DateTimeImmutab
369399
$current_date->setTime(0, 0, 0);
370400
}
371401
}
402+
403+
protected function getCacheFilename(string $base_dir, DateTimeImmutable $start, DateTimeImmutable $end): string
404+
{
405+
return sprintf(
406+
'%s/%s_%s.json',
407+
$base_dir,
408+
$start->format('Y-m-d'),
409+
$end->format('Y-m-d')
410+
);
411+
}
372412
}

‎src/DataSource/CarbonIntensity/Rte/Client.php‎

Lines changed: 87 additions & 68 deletions
Original file line numberDiff line numberDiff line change
@@ -59,13 +59,13 @@ class Client extends AbstractClient
5959
const EXPORT_URL_REALTIME = '/eco2mix-national-tr/exports/json';
6060
const EXPORT_URL_CONSOLIDATED = '/eco2mix-national-cons-def/exports/json';
6161

62+
const DATASET_REALTIME = 0;
63+
const DATASET_CONSOLIDATED = 1;
64+
6265
private RestApiClientInterface $client;
6366

6467
private string $base_url;
6568

66-
/** @var bool Use consolidated dataset (true) or realtime dataset (false) */
67-
private bool $use_consolidated = false;
68-
6969
public function __construct(RestApiClientInterface $client, string $url = '')
7070
{
7171
$this->client = $client;
@@ -208,91 +208,100 @@ public function fetchDay(DateTimeImmutable $day, string $zone): array
208208
}
209209

210210
/**
211-
* Fetch carbon intensities from Opendata Réseaux-Énergies using export dataset.
212-
*
213-
* See https://odre.opendatasoft.com/explore/dataset/eco2mix-national-tr/api/?disjunctive.nature
211+
* Fetch range from cached data or online database. Assume that $start is the beginning of a month (Y-m-1 00:00:00)
212+
* and $stop is the beginning of the next month (Y-m+1-1 00:00:00).
214213
*
215-
* The method fetches the intensities for the date range specified in argument.
216214
* @param DateTimeImmutable $start
217215
* @param DateTimeImmutable $stop
218216
* @param string $zone
217+
* @param string $dataset
219218
* @return array
220219
*/
221-
public function fetchRange(DateTimeImmutable $start, DateTimeImmutable $stop, string $zone): array
220+
public function fetchRange(DateTimeImmutable $start, DateTimeImmutable $stop, string $zone, string $dataset = self::DATASET_REALTIME): array
222221
{
223222
/** @var DBmysql $DB */
224223
global $DB;
225224

226-
$format = DateTime::ATOM;
227-
$from = $start->format($format);
228-
$to = $stop->format($format);
225+
// Build realtime and consolidated paths
226+
$base_path = GLPI_PLUGIN_DOC_DIR . '/carbon/carbon_intensity/' . $this->getSourceName() . '/' . $zone;
227+
$consolidated_dir = $base_path . '/consolidated';
228+
$realtime_dir = $base_path . '/realtime';
229229

230+
// Set timezone to +00:00 and extend range by 12 hours on each side
231+
$request_start = $start->setTimezone(new DateTimeZone('+0000'))->sub(new DateInterval('PT12H'));
232+
$request_stop = $stop->setTimezone(new DateTimeZone('+0000'))->add(new DateInterval('PT12H'));
233+
$format = DateTime::ATOM;
234+
$from = $request_start->format($format);
235+
$to = $request_stop->format($format);
236+
$interval = $request_stop->diff($request_start);
237+
$expected_samples_count = (int) ($interval->days * 24)
238+
+ (int) ($interval->h)
239+
+ (int) ($interval->i / 60);
230240
$timezone = $DB->guessTimezone();
231-
$interval = $stop->diff($start);
232-
$where = "date_heure IN [date'$from' TO date'$to'";
233-
if ($interval->y === 0 && $interval->m === 0 && $interval->d === 0 && $interval->h === 1) {
234-
$where .= ']';
235-
} else {
236-
$where .= '[';
237-
}
241+
$where = "date_heure IN [date'$from' TO date'$to'[ AND taux_co2 is not null";
238242
$params = [
239-
'select' => 'taux_co2,date_heure',
243+
'select' => 'date_heure,taux_co2',
240244
'where' => $where,
241245
'order_by' => 'date_heure asc',
242-
'timezone' => $timezone,
246+
'timezone' => '+0000'
243247
];
244-
// convert to 15 minutes interval
245-
if ($this->use_consolidated) {
246-
$this->step = 60;
247-
$url = $this->base_url . self::EXPORT_URL_CONSOLIDATED;
248-
} else {
249-
$this->step = 15;
250-
$url = $this->base_url . self::EXPORT_URL_REALTIME;
248+
249+
// Prepend base URL
250+
switch ($dataset) {
251+
case self::DATASET_CONSOLIDATED:
252+
$url = self::EXPORT_URL_CONSOLIDATED;
253+
$cache_file = $this->getCacheFilename(
254+
$consolidated_dir,
255+
$start,
256+
$stop
257+
);
258+
break;
259+
case self::DATASET_REALTIME:
260+
default:
261+
$url = self::EXPORT_URL_REALTIME;
262+
$cache_file = $this->getCacheFilename(
263+
$realtime_dir,
264+
$start,
265+
$stop
266+
);
267+
break;
251268
}
252-
$expected_samples_count = (int) ($interval->days * 24)
253-
+ (int) ($interval->h)
254-
+ (int) ($interval->i / 60);
255269

256-
$alt_response = [];
257-
$response = $this->client->request('GET', $url, ['timeout' => 8, 'query' => $params]);
258-
if ($response) {
259-
// Adjust step as consolidated dataset may return data for each 15 minutes !
270+
// If cached file exists, use it
271+
if (file_exists($cache_file)) {
272+
$response = json_decode(file_get_contents($cache_file), true);
260273
$this->step = $this->detectStep($response);
261-
$expected_samples_count *= (60 / $this->step);
274+
return $response;
262275
}
276+
@mkdir(dirname($cache_file), 0755, true);
263277

264-
// Tolerate DST switching issues with 15 minutes samples (4 missing samples or too many samples)
265-
if (!$response || abs(count($response) - $expected_samples_count) > 4) {
266-
// Retry with realtime dataset
267-
if (!$this->use_consolidated) {
268-
$this->use_consolidated = true;
269-
$alt_response = $this->fetchRange($start, $stop, $zone);
270-
271-
if (!isset($alt_response['error_code']) && count($alt_response) > count($response)) {
272-
// Use the alternative response if more samples than the original response
273-
$response = $alt_response;
274-
}
278+
$url = $this->base_url . $url;
279+
$response = $this->client->request('GET', $url, ['timeout' => 8, 'query' => $params]);
280+
$this->step = $this->detectStep($response);
281+
$expected_samples_count *= (60 / $this->step);
282+
if (!$response || ($dataset === self::DATASET_REALTIME && abs(count($response) - $expected_samples_count) > 4)) {
283+
$alt_response = $this->fetchRange($start, $stop, $zone, self::DATASET_CONSOLIDATED);
284+
if (!isset($alt_response['error_code']) && count($alt_response) > count($response)) {
285+
// Use the alternative response if more samples than the original response
286+
$response = $alt_response;
275287
}
288+
} else {
289+
$json = json_encode($response);
290+
file_put_contents($cache_file, $json);
276291
}
277-
278-
if (!$response) {
279-
trigger_error('No response from RTE API for ' . $zone, E_USER_WARNING);
280-
return [];
281-
}
282-
if (count($response) === 0) {
283-
trigger_error('Empty response from RTE API for ' . $zone, E_USER_WARNING);
284-
return [];
285-
}
286-
if (abs(count($response) - $expected_samples_count) > 4) {
287-
trigger_error('Not enough samples from RTE API for ' . $zone . ' (expected: ' . $expected_samples_count . ', got: ' . count($response) . ')', E_USER_WARNING);
288-
}
289-
if (isset($response['error_code'])) {
290-
trigger_error($this->formatError($response));
291-
}
292-
293292
return $response;
294293
}
295294

295+
protected function getCacheFilename(string $base_dir, DateTimeImmutable $start, DateTimeImmutable $end): string
296+
{
297+
return sprintf(
298+
'%s/%s_%s.json',
299+
$base_dir,
300+
$start->format('Y-m-d'),
301+
$end->format('Y-m-d')
302+
);
303+
}
304+
296305
protected function formatOutput(array $response, int $step): array
297306
{
298307
/** @var DBMysql $DB */
@@ -309,11 +318,13 @@ protected function formatOutput(array $response, int $step): array
309318
// Even if we use UTC timezone.
310319
$filtered_response = $this->deduplicate($response);
311320

312-
// Convert string dates into datetime objects, restoring timezone as type Continent/City instead of offset
321+
// Convert string dates into datetime objects, shifting to local timezone
322+
// and using timezone expressed as type Continent/City instead of offset
313323
// This is needed to detect later the switching to winter time
314-
$timezone = new DateTimeZone($DB->guessTimezone());
324+
$timezone = new DateTimeZone('+0000');
325+
$local_timezone = new DateTimeZone($DB->guessTimezone());
315326
foreach ($filtered_response as &$record) {
316-
$record['date_heure'] = DateTime::createFromFormat('Y-m-d\TH:i:s??????', $record['date_heure'], $timezone);
327+
$record['date_heure'] = DateTime::createFromFormat('Y-m-d\TH:i:s??????', $record['date_heure'], $timezone)->setTimezone($local_timezone);
317328
}
318329

319330
// Convert samples from 15 min to 1 hour
@@ -354,6 +365,12 @@ protected function deduplicate(array $records): array
354365
return $filtered_response;
355366
}
356367

368+
/**
369+
* Get the temporal distance between records
370+
*
371+
* @param array $records
372+
* @return integer step in minutes
373+
*/
357374
protected function detectStep(array $records): ?int
358375
{
359376
if (count($records) < 2) {
@@ -366,9 +383,8 @@ protected function detectStep(array $records): ?int
366383

367384
if ($diff->h === 1) {
368385
return 60; // 1 hour step
369-
} else {
370-
return $diff->i; // Return the minutes step
371386
}
387+
return $diff->i; // Return the minutes step
372388
}
373389

374390
/**
@@ -387,8 +403,11 @@ protected function convertToHourly(array $records, int $step): array
387403

388404
foreach ($records as $record) {
389405
$date = $record['date_heure'];
390-
$count++;
391406
$intensity += $record['taux_co2'];
407+
if ($record['taux_co2'] === null) {
408+
continue;
409+
}
410+
$count++;
392411
$minute = (int) $date->format('i');
393412

394413
if ($previous_record_date !== null) {

0 commit comments

Comments
 (0)