Skip to content

Commit a3d3122

Browse files
btrybtry
committed
feat(DataSource\CarbonIntensity): response caching
1 parent 9379517 commit a3d3122

File tree

9 files changed

+263
-87
lines changed

9 files changed

+263
-87
lines changed

‎src/CarbonIntensity.php‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -213,7 +213,7 @@ public function getFirstKnownDate(string $zone_name, string $source_name): ?Date
213213
*/
214214
public function downloadOneZone(ClientInterface $data_source, string $zone_name, int $limit = 0, ?ProgressBar $progress_bar = null): int
215215
{
216-
$start_date = $this->getDownloadStartDate($zone_name, $data_source);
216+
$start_date = $this->getDownloadStartDate();
217217

218218
$total_count = 0;
219219

@@ -279,11 +279,9 @@ public function downloadOneZone(ClientInterface $data_source, string $zone_name,
279279
/**
280280
* Get the oldest date where data are required
281281
*
282-
* @param string $zone_name ignored for now; zone to examine
283-
* @param ClientInterface $data_source ignored for now; data source
284282
* @return DateTimeImmutable|null
285283
*/
286-
public function getDownloadStartDate(string $zone_name, ClientInterface $data_source): ?DateTimeImmutable
284+
public function getDownloadStartDate(): ?DateTimeImmutable
287285
{
288286
// Get the default oldest date od data to download
289287
$start_date = new DateTime(self::MIN_HISTORY_LENGTH);

‎src/Command/CollectCarbonIntensityCommand.php‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -169,7 +169,7 @@ protected function execute(InputInterface $input, OutputInterface $output)
169169
$carbon_intensity->downloadOneZone($this->client, $this->zones[$zone_code], 0, new ProgressBar($this->output));
170170

171171
// Find start and stop dates to cover
172-
$start_date = $carbon_intensity->getDownloadStartDate($this->zones[$zone_code], $this->client);
172+
$start_date = $carbon_intensity->getDownloadStartDate();
173173
$gaps = $carbon_intensity->findGaps($this->source_id, $zone->getID(), $start_date);
174174

175175
// Count the hours not covered by any sample

‎src/DataSource/CarbonIntensity/AbstractClient.php‎

Lines changed: 21 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -157,8 +157,17 @@ public function getZones(array $crit = []): array
157157

158158
public function fullDownload(string $zone, DateTimeImmutable $start_date, DateTimeImmutable $stop_date, CarbonIntensity $intensity, int $limit = 0, ?ProgressBar $progress_bar = null): int
159159
{
160+
if ($start_date >= $stop_date) {
161+
return 0;
162+
}
163+
// Round start date to beginning of month
164+
$start_date = $start_date->setTime(0, 0, 0, 0)->setDate((int) $start_date->format('Y'), (int) $start_date->format('m'), 1);
165+
$stop_date = $stop_date->setTime(0, 0, 0, 0)->setDate((int) $stop_date->format('Y'), (int) $stop_date->format('m'), 1);
160166
$count = 0;
161167
$saved = 0;
168+
if ($start_date == $stop_date) {
169+
$stop_date = $stop_date->add(new DateInterval('P1M'));
170+
}
162171

163172
/**
164173
* Huge quantity of SQL queries will be executed
@@ -171,9 +180,18 @@ public function fullDownload(string $zone, DateTimeImmutable $start_date, DateTi
171180
// enpty string in PHPUnit environment
172181
$memory_limit = null;
173182
}
174-
foreach ($this->sliceDateRangeByMonth($start_date, $stop_date) as $slice) {
183+
184+
// Traverse each month from start_date to end_date
185+
$current_date = DateTime::createFromImmutable($start_date);
186+
while ($current_date < $stop_date) {
187+
$next_month = clone $current_date;
188+
$next_month->add(new DateInterval('P1M'));
175189
try {
176-
$data = $this->fetchRange($slice['start'], $slice['stop'], $zone);
190+
$data = $this->fetchRange(
191+
DateTimeImmutable::createFromMutable($current_date),
192+
DateTimeImmutable::createFromMutable($next_month),
193+
$zone
194+
);
177195
} catch (AbortException $e) {
178196
break;
179197
}
@@ -194,8 +212,8 @@ public function fullDownload(string $zone, DateTimeImmutable $start_date, DateTi
194212
// 8 MB memory left, emergency exit
195213
return $saved > 0 ? $count : -$count;
196214
}
215+
$current_date = $next_month;
197216
}
198-
199217
return $saved > 0 ? $count : -$count;
200218
}
201219

‎src/DataSource/CarbonIntensity/ElectricityMaps/Client.php‎

Lines changed: 45 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,6 @@
4141
use GlpiPlugin\Carbon\Source;
4242
use GlpiPlugin\Carbon\Zone;
4343
use GlpiPlugin\Carbon\Source_Zone;
44-
use Config as GlpiConfig;
4544
use GLPIKey;
4645
use GlpiPlugin\Carbon\DataSource\CarbonIntensity\AbortException;
4746
use GlpiPlugin\Carbon\DataSource\CarbonIntensity\AbstractClient;
@@ -264,16 +263,45 @@ public function fetchDay(DateTimeImmutable $day, string $zone): array
264263

265264
public function fetchRange(DateTimeImmutable $start, DateTimeImmutable $stop, string $zone): array
266265
{
267-
268266
// TODO: get zones from GLPI locations
269267
$params = [
270268
'zone' => $zone,
271269
];
272270

271+
$base_path = GLPI_PLUGIN_DOC_DIR . '/carbon/carbon_intensity/' . $this->getSourceName() . '/' . $zone;
272+
$cache_file = $this->getCacheFilename(
273+
$base_path,
274+
$start,
275+
$stop
276+
);
277+
// If cached file exists, use it
278+
if (file_exists($cache_file)) {
279+
$full_response = json_decode(file_get_contents($cache_file), true);
280+
return $full_response;
281+
}
282+
@mkdir(dirname($cache_file), 0755, true);
283+
284+
// Set timezone to +00:00 and extend range by 12 hours on each side
285+
$request_start = $start->setTimezone(new DateTimeZone('+0000'))->sub(new DateInterval('PT12H'));
286+
$request_stop = $stop->setTimezone(new DateTimeZone('+0000'))->add(new DateInterval('PT12H'));
273287
$this->step = 60;
274288

275-
$response = $this->client->request('GET', $this->base_url . self::PAST_URL, ['query' => $params]);
276-
if (!$response) {
289+
$step = new DateInterval('P10D');
290+
$full_response = [];
291+
$current_date = DateTime::createFromImmutable($request_start);
292+
while ($current_date < $request_stop) {
293+
$response = $this->client->request('GET', $this->base_url . self::PAST_URL, ['query' => $params]);
294+
if (!$full_response) {
295+
$full_response = $response;
296+
} else {
297+
$full_response['data'] = array_merge($full_response['data'], $response['data']);
298+
}
299+
$current_date->add($step);
300+
if ($current_date > $request_stop) {
301+
$current_date = $request_stop;
302+
}
303+
}
304+
if (!$full_response) {
277305
return [];
278306
}
279307

@@ -286,7 +314,9 @@ public function fetchRange(DateTimeImmutable $start, DateTimeImmutable $stop, st
286314
return [];
287315
}
288316

289-
return $$response['history'];
317+
$json = json_encode($full_response);
318+
file_put_contents($cache_file, $json);
319+
return $full_response['data'];
290320
}
291321

292322
protected function formatOutput(array $response, int $step): array
@@ -369,4 +399,14 @@ protected function sliceDateRangeByDay(DateTimeImmutable $start, DateTimeImmutab
369399
$current_date->setTime(0, 0, 0);
370400
}
371401
}
402+
403+
protected function getCacheFilename(string $base_dir, DateTimeImmutable $start, DateTimeImmutable $end): string
404+
{
405+
return sprintf(
406+
'%s/%s_%s.json',
407+
$base_dir,
408+
$start->format('Y-m-d'),
409+
$end->format('Y-m-d')
410+
);
411+
}
372412
}

‎src/DataSource/CarbonIntensity/Rte/Client.php‎

Lines changed: 87 additions & 72 deletions
Original file line numberDiff line numberDiff line change
@@ -59,13 +59,13 @@ class Client extends AbstractClient
5959
const EXPORT_URL_REALTIME = '/eco2mix-national-tr/exports/json';
6060
const EXPORT_URL_CONSOLIDATED = '/eco2mix-national-cons-def/exports/json';
6161

62+
const DATASET_REALTIME = 0;
63+
const DATASET_CONSOLIDATED = 1;
64+
6265
private RestApiClientInterface $client;
6366

6467
private string $base_url;
6568

66-
/** @var bool Use consolidated dataset (true) or realtime dataset (false) */
67-
private bool $use_consolidated = false;
68-
6969
public function __construct(RestApiClientInterface $client, string $url = '')
7070
{
7171
$this->client = $client;
@@ -208,91 +208,96 @@ public function fetchDay(DateTimeImmutable $day, string $zone): array
208208
}
209209

210210
/**
211-
* Fetch carbon intensities from Opendata Réseaux-Énergies using export dataset.
212-
*
213-
* See https://odre.opendatasoft.com/explore/dataset/eco2mix-national-tr/api/?disjunctive.nature
211+
* Fetch range from cached data or online database. Assume that $start is the beginning of a month (Y-m-1 00:00:00)
212+
* and $stop is the beginning of the next month (Y-m+1-1 00:00:00).
214213
*
215-
* The method fetches the intensities for the date range specified in argument.
216214
* @param DateTimeImmutable $start
217215
* @param DateTimeImmutable $stop
218216
* @param string $zone
217+
* @param int $dataset
219218
* @return array
220219
*/
221-
public function fetchRange(DateTimeImmutable $start, DateTimeImmutable $stop, string $zone): array
220+
public function fetchRange(DateTimeImmutable $start, DateTimeImmutable $stop, string $zone, int $dataset = self::DATASET_REALTIME): array
222221
{
223-
/** @var DBmysql $DB */
224-
global $DB;
225-
222+
// Build realtime and consolidated paths
223+
$base_path = GLPI_PLUGIN_DOC_DIR . '/carbon/carbon_intensity/' . $this->getSourceName() . '/' . $zone;
224+
$consolidated_dir = $base_path . '/consolidated';
225+
$realtime_dir = $base_path . '/realtime';
226+
227+
// Set timezone to +00:00 and extend range by 12 hours on each side
228+
$request_start = $start->setTimezone(new DateTimeZone('+0000'))->sub(new DateInterval('PT12H'));
229+
$request_stop = $stop->setTimezone(new DateTimeZone('+0000'))->add(new DateInterval('PT12H'));
226230
$format = DateTime::ATOM;
227-
$from = $start->format($format);
228-
$to = $stop->format($format);
229-
230-
$timezone = $DB->guessTimezone();
231-
$interval = $stop->diff($start);
232-
$where = "date_heure IN [date'$from' TO date'$to'";
233-
if ($interval->y === 0 && $interval->m === 0 && $interval->d === 0 && $interval->h === 1) {
234-
$where .= ']';
235-
} else {
236-
$where .= '[';
237-
}
231+
$from = $request_start->format($format);
232+
$to = $request_stop->format($format);
233+
$interval = $request_stop->diff($request_start);
234+
$expected_samples_count = (int) ($interval->days * 24)
235+
+ (int) ($interval->h)
236+
+ (int) ($interval->i / 60);
237+
$where = "date_heure IN [date'$from' TO date'$to'[ AND taux_co2 is not null";
238238
$params = [
239-
'select' => 'taux_co2,date_heure',
239+
'select' => 'date_heure,taux_co2',
240240
'where' => $where,
241241
'order_by' => 'date_heure asc',
242-
'timezone' => $timezone,
242+
'timezone' => '+0000'
243243
];
244-
// convert to 15 minutes interval
245-
if ($this->use_consolidated) {
246-
$this->step = 60;
247-
$url = $this->base_url . self::EXPORT_URL_CONSOLIDATED;
248-
} else {
249-
$this->step = 15;
250-
$url = $this->base_url . self::EXPORT_URL_REALTIME;
244+
245+
// Prepend base URL
246+
switch ($dataset) {
247+
case self::DATASET_CONSOLIDATED:
248+
$url = self::EXPORT_URL_CONSOLIDATED;
249+
$cache_file = $this->getCacheFilename(
250+
$consolidated_dir,
251+
$start,
252+
$stop
253+
);
254+
break;
255+
case self::DATASET_REALTIME:
256+
default:
257+
$url = self::EXPORT_URL_REALTIME;
258+
$cache_file = $this->getCacheFilename(
259+
$realtime_dir,
260+
$start,
261+
$stop
262+
);
263+
break;
251264
}
252-
$expected_samples_count = (int) ($interval->days * 24)
253-
+ (int) ($interval->h)
254-
+ (int) ($interval->i / 60);
255265

256-
$alt_response = [];
257-
$response = $this->client->request('GET', $url, ['timeout' => 8, 'query' => $params]);
258-
if ($response) {
259-
// Adjust step as consolidated dataset may return data for each 15 minutes !
266+
// If cached file exists, use it
267+
if (file_exists($cache_file)) {
268+
$response = json_decode(file_get_contents($cache_file), true);
260269
$this->step = $this->detectStep($response);
261-
$expected_samples_count *= (60 / $this->step);
270+
return $response;
262271
}
272+
@mkdir(dirname($cache_file), 0755, true);
263273

264-
// Tolerate DST switching issues with 15 minutes samples (4 missing samples or too many samples)
265-
if (!$response || abs(count($response) - $expected_samples_count) > 4) {
266-
// Retry with realtime dataset
267-
if (!$this->use_consolidated) {
268-
$this->use_consolidated = true;
269-
$alt_response = $this->fetchRange($start, $stop, $zone);
270-
271-
if (!isset($alt_response['error_code']) && count($alt_response) > count($response)) {
272-
// Use the alternative response if more samples than the original response
273-
$response = $alt_response;
274-
}
274+
$url = $this->base_url . $url;
275+
$response = $this->client->request('GET', $url, ['timeout' => 8, 'query' => $params]);
276+
$this->step = $this->detectStep($response);
277+
$expected_samples_count *= (60 / $this->step);
278+
if (!$response || ($dataset === self::DATASET_REALTIME && abs(count($response) - $expected_samples_count) > 4)) {
279+
$alt_response = $this->fetchRange($start, $stop, $zone, self::DATASET_CONSOLIDATED);
280+
if (!isset($alt_response['error_code']) && count($alt_response) > count($response)) {
281+
// Use the alternative response if more samples than the original response
282+
$response = $alt_response;
275283
}
284+
} else {
285+
$json = json_encode($response);
286+
file_put_contents($cache_file, $json);
276287
}
277-
278-
if (!$response) {
279-
trigger_error('No response from RTE API for ' . $zone, E_USER_WARNING);
280-
return [];
281-
}
282-
if (count($response) === 0) {
283-
trigger_error('Empty response from RTE API for ' . $zone, E_USER_WARNING);
284-
return [];
285-
}
286-
if (abs(count($response) - $expected_samples_count) > 4) {
287-
trigger_error('Not enough samples from RTE API for ' . $zone . ' (expected: ' . $expected_samples_count . ', got: ' . count($response) . ')', E_USER_WARNING);
288-
}
289-
if (isset($response['error_code'])) {
290-
trigger_error($this->formatError($response));
291-
}
292-
293288
return $response;
294289
}
295290

291+
protected function getCacheFilename(string $base_dir, DateTimeImmutable $start, DateTimeImmutable $end): string
292+
{
293+
return sprintf(
294+
'%s/%s_%s.json',
295+
$base_dir,
296+
$start->format('Y-m-d'),
297+
$end->format('Y-m-d')
298+
);
299+
}
300+
296301
protected function formatOutput(array $response, int $step): array
297302
{
298303
/** @var DBMysql $DB */
@@ -309,11 +314,13 @@ protected function formatOutput(array $response, int $step): array
309314
// Even if we use UTC timezone.
310315
$filtered_response = $this->deduplicate($response);
311316

312-
// Convert string dates into datetime objects, restoring timezone as type Continent/City instead of offset
317+
// Convert string dates into datetime objects, shifting to local timezone
318+
// and using timezone expressed as type Continent/City instead of offset
313319
// This is needed to detect later the switching to winter time
314-
$timezone = new DateTimeZone($DB->guessTimezone());
320+
$timezone = new DateTimeZone('+0000');
321+
$local_timezone = new DateTimeZone($DB->guessTimezone());
315322
foreach ($filtered_response as &$record) {
316-
$record['date_heure'] = DateTime::createFromFormat('Y-m-d\TH:i:s??????', $record['date_heure'], $timezone);
323+
$record['date_heure'] = DateTime::createFromFormat('Y-m-d\TH:i:s??????', $record['date_heure'], $timezone)->setTimezone($local_timezone);
317324
}
318325

319326
// Convert samples from 15 min to 1 hour
@@ -354,6 +361,12 @@ protected function deduplicate(array $records): array
354361
return $filtered_response;
355362
}
356363

364+
/**
365+
* Get the temporal distance between records
366+
*
367+
* @param array $records
368+
* @return integer step in minutes
369+
*/
357370
protected function detectStep(array $records): ?int
358371
{
359372
if (count($records) < 2) {
@@ -366,9 +379,8 @@ protected function detectStep(array $records): ?int
366379

367380
if ($diff->h === 1) {
368381
return 60; // 1 hour step
369-
} else {
370-
return $diff->i; // Return the minutes step
371382
}
383+
return $diff->i; // Return the minutes step
372384
}
373385

374386
/**
@@ -387,8 +399,11 @@ protected function convertToHourly(array $records, int $step): array
387399

388400
foreach ($records as $record) {
389401
$date = $record['date_heure'];
390-
$count++;
391402
$intensity += $record['taux_co2'];
403+
if ($record['taux_co2'] === null) {
404+
continue;
405+
}
406+
$count++;
392407
$minute = (int) $date->format('i');
393408

394409
if ($previous_record_date !== null) {

0 commit comments

Comments
 (0)