From 309ba7bab8972a1e6f60ebefb614297bbff972f7 Mon Sep 17 00:00:00 2001 From: Thiago Novaes Date: Wed, 30 Oct 2024 09:14:52 +0100 Subject: [PATCH 01/10] urls --- src/ElectricityMarketData.jl | 3 ++ src/caiso/urls.jl | 53 ++++++++++++++++++++++++++++++++++++ test/caiso/urls.jl | 38 ++++++++++++++++++++++++++ test/runtests.jl | 2 ++ 4 files changed, 96 insertions(+) create mode 100644 src/caiso/urls.jl create mode 100644 test/caiso/urls.jl diff --git a/src/ElectricityMarketData.jl b/src/ElectricityMarketData.jl index c303673..07f2570 100644 --- a/src/ElectricityMarketData.jl +++ b/src/ElectricityMarketData.jl @@ -37,4 +37,7 @@ include("pjm/urls.jl") include("pjm/utils.jl") include("pjm/parser.jl") +#caiso +include("caiso/urls.jl") + end # module diff --git a/src/caiso/urls.jl b/src/caiso/urls.jl new file mode 100644 index 0000000..4d08dc7 --- /dev/null +++ b/src/caiso/urls.jl @@ -0,0 +1,53 @@ +""" + _get_caiso_url(queryname::AbstractString, startdate::AbstractString, stopdate::AbstractString, market_run_id::AbstractString, version::AbstractString = "1", resultformat::AbstractString="6")::AbstractString + +Returns the url for the Caiso API. +""" +function _get_caiso_url( + queryname::AbstractString, + startdate::AbstractString, + stopdate::AbstractString, + market_run_id::AbstractString, + version::AbstractString = "1", + resultformat::AbstractString = "6", +)::AbstractString + return "http://oasis.caiso.com/oasisapi/SingleZip?resultformat=$resultformat&queryname=$queryname&version=$version&startdatetime=$startdate&enddatetime=$stopdate&market_run_id=$market_run_id" +end + +""" + _get_caiso_price_url(startdate::AbstractString, stopdate::AbstractString, market_run_id::AbstractString)::AbstractString + +Returns the price url for the Caiso API. +""" +function _get_caiso_price_url( + startdate::AbstractString, + stopdate::AbstractString, + market_run_id::AbstractString, +)::AbstractString + return _get_caiso_url("PRC_LMP", startdate, stopdate, market_run_id) +end + +""" + _get_caiso_real_time_url(startdate::AbstractString, stopdate::AbstractString)::AbstractString + +Returns the real time url for the Caiso API. +""" +function _get_caiso_real_time_url( + startdate::AbstractString, + stopdate::AbstractString, +)::AbstractString + return _get_caiso_price_url(startdate, stopdate, "RTM") +end + +""" + _get_caiso_day_ahead_url(startdate::AbstractString, stopdate::AbstractString)::AbstractString + +Returns the day ahead url for the Caiso API. +""" +function _get_caiso_day_ahead_url( + startdate::AbstractString, + stopdate::AbstractString, +)::AbstractString + return _get_caiso_price_url(startdate, stopdate, "DAM") +end + diff --git a/test/caiso/urls.jl b/test/caiso/urls.jl new file mode 100644 index 0000000..6e38430 --- /dev/null +++ b/test/caiso/urls.jl @@ -0,0 +1,38 @@ +@testset "caiso/urls.jl" begin + @testset "_get_caiso_url" begin + queryname = "PRC_LMP" + startdate = "20210105T07:00-0000" + stopdate = "20210106T07:00-0000" + market_run_id = "RTM" + url = ElectricityMarketData._get_caiso_url( + queryname, + startdate, + stopdate, + market_run_id, + ) + @test url == + "http://oasis.caiso.com/oasisapi/SingleZip?resultformat=6&queryname=PRC_LMP&version=1&startdatetime=20210105T07:00-0000&enddatetime=20210106T07:00-0000&market_run_id=RTM" + end + @testset "_get_caiso_price_url" begin + startdate = "20230104T07:00-0000" + stopdate = "20230105T07:00-0000" + market_run_id = "RTM" + url = ElectricityMarketData._get_caiso_price_url(startdate, stopdate, market_run_id) + @test url == + "http://oasis.caiso.com/oasisapi/SingleZip?resultformat=6&queryname=PRC_LMP&version=1&startdatetime=20230104T07:00-0000&enddatetime=20230105T07:00-0000&market_run_id=RTM" + end + @testset "_get_caiso_real_time_url" begin + startdate = "20230104T07:00-0300" + stopdate = "20230105T07:00-0300" + url = ElectricityMarketData._get_caiso_real_time_url(startdate, stopdate) + @test url == + "http://oasis.caiso.com/oasisapi/SingleZip?resultformat=6&queryname=PRC_LMP&version=1&startdatetime=20230104T07:00-0300&enddatetime=20230105T07:00-0300&market_run_id=RTM" + end + @testset "_get_caiso_day_ahead_url" begin + startdate = "20230104T07:00-0300" + stopdate = "20230105T07:00-0000" + url = ElectricityMarketData._get_caiso_day_ahead_url(startdate, stopdate) + @test url == + "http://oasis.caiso.com/oasisapi/SingleZip?resultformat=6&queryname=PRC_LMP&version=1&startdatetime=20230104T07:00-0300&enddatetime=20230105T07:00-0000&market_run_id=DAM" + end +end diff --git a/test/runtests.jl b/test/runtests.jl index 41553b1..a6e0811 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -18,3 +18,5 @@ include("pjm/pjm_market.jl") include("pjm/urls.jl") include("pjm/utils.jl") include("pjm/parser.jl") +#caiso +include("caiso/urls.jl") From 4bb0c5c749f2ed8a8de09e7214f2663b7f2b7b8f Mon Sep 17 00:00:00 2001 From: Thiago Novaes Date: Wed, 30 Oct 2024 11:05:18 +0100 Subject: [PATCH 02/10] datetime --- src/ElectricityMarketData.jl | 1 + src/caiso/datetime.jl | 11 +++++++++++ test/caiso/datetime.jl | 7 +++++++ test/runtests.jl | 1 + 4 files changed, 20 insertions(+) create mode 100644 src/caiso/datetime.jl create mode 100644 test/caiso/datetime.jl diff --git a/src/ElectricityMarketData.jl b/src/ElectricityMarketData.jl index 07f2570..ff9fb33 100644 --- a/src/ElectricityMarketData.jl +++ b/src/ElectricityMarketData.jl @@ -38,6 +38,7 @@ include("pjm/utils.jl") include("pjm/parser.jl") #caiso +include("caiso/datetime.jl") include("caiso/urls.jl") end # module diff --git a/src/caiso/datetime.jl b/src/caiso/datetime.jl new file mode 100644 index 0000000..1437397 --- /dev/null +++ b/src/caiso/datetime.jl @@ -0,0 +1,11 @@ +""" + _get_zdt_formated(zdt::ZonedDateTime)::AbstractString + +Get the zdt formated string in the format "yyyymmddTHH:MMzzzz". +""" +function _get_zdt_formated(zdt::ZonedDateTime)::AbstractString + return string( + Dates.format(zdt, "yyyymmddTHH:MM"), + replace(Dates.format(zdt, "zzzz"), ":" => ""), + ) +end diff --git a/test/caiso/datetime.jl b/test/caiso/datetime.jl new file mode 100644 index 0000000..1997838 --- /dev/null +++ b/test/caiso/datetime.jl @@ -0,0 +1,7 @@ +@testset "caiso/date.jl" begin + @testset "_get_zdt_formated" begin + zdt = ZonedDateTime(2023, 5, 4, 7, 2, 13, tz"UTC-3") + formated = ElectricityMarketData._get_zdt_formated(zdt) + @test formated == "20230504T07:02-0300" + end +end diff --git a/test/runtests.jl b/test/runtests.jl index a6e0811..fe2d9d5 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -19,4 +19,5 @@ include("pjm/urls.jl") include("pjm/utils.jl") include("pjm/parser.jl") #caiso +include("caiso/datetime.jl") include("caiso/urls.jl") From fab86c339831fcab8868866dd3b690e97cf68fe2 Mon Sep 17 00:00:00 2001 From: Thiago Novaes Date: Wed, 30 Oct 2024 11:28:36 +0100 Subject: [PATCH 03/10] fix url --- src/caiso/urls.jl | 2 +- test/caiso/urls.jl | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/caiso/urls.jl b/src/caiso/urls.jl index 4d08dc7..b5a68eb 100644 --- a/src/caiso/urls.jl +++ b/src/caiso/urls.jl @@ -24,7 +24,7 @@ function _get_caiso_price_url( stopdate::AbstractString, market_run_id::AbstractString, )::AbstractString - return _get_caiso_url("PRC_LMP", startdate, stopdate, market_run_id) + return _get_caiso_url("PRC_INTVL_LMP", startdate, stopdate, market_run_id) end """ diff --git a/test/caiso/urls.jl b/test/caiso/urls.jl index 6e38430..0617d9e 100644 --- a/test/caiso/urls.jl +++ b/test/caiso/urls.jl @@ -1,6 +1,6 @@ @testset "caiso/urls.jl" begin @testset "_get_caiso_url" begin - queryname = "PRC_LMP" + queryname = "PRC_INTVL_LMP" startdate = "20210105T07:00-0000" stopdate = "20210106T07:00-0000" market_run_id = "RTM" @@ -11,7 +11,7 @@ market_run_id, ) @test url == - "http://oasis.caiso.com/oasisapi/SingleZip?resultformat=6&queryname=PRC_LMP&version=1&startdatetime=20210105T07:00-0000&enddatetime=20210106T07:00-0000&market_run_id=RTM" + "http://oasis.caiso.com/oasisapi/SingleZip?resultformat=6&queryname=PRC_INTVL_LMP&version=1&startdatetime=20210105T07:00-0000&enddatetime=20210106T07:00-0000&market_run_id=RTM" end @testset "_get_caiso_price_url" begin startdate = "20230104T07:00-0000" @@ -19,20 +19,20 @@ market_run_id = "RTM" url = ElectricityMarketData._get_caiso_price_url(startdate, stopdate, market_run_id) @test url == - "http://oasis.caiso.com/oasisapi/SingleZip?resultformat=6&queryname=PRC_LMP&version=1&startdatetime=20230104T07:00-0000&enddatetime=20230105T07:00-0000&market_run_id=RTM" + "http://oasis.caiso.com/oasisapi/SingleZip?resultformat=6&queryname=PRC_INTVL_LMP&version=1&startdatetime=20230104T07:00-0000&enddatetime=20230105T07:00-0000&market_run_id=RTM" end @testset "_get_caiso_real_time_url" begin startdate = "20230104T07:00-0300" stopdate = "20230105T07:00-0300" url = ElectricityMarketData._get_caiso_real_time_url(startdate, stopdate) @test url == - "http://oasis.caiso.com/oasisapi/SingleZip?resultformat=6&queryname=PRC_LMP&version=1&startdatetime=20230104T07:00-0300&enddatetime=20230105T07:00-0300&market_run_id=RTM" + "http://oasis.caiso.com/oasisapi/SingleZip?resultformat=6&queryname=PRC_INTVL_LMP&version=1&startdatetime=20230104T07:00-0300&enddatetime=20230105T07:00-0300&market_run_id=RTM" end @testset "_get_caiso_day_ahead_url" begin startdate = "20230104T07:00-0300" stopdate = "20230105T07:00-0000" url = ElectricityMarketData._get_caiso_day_ahead_url(startdate, stopdate) @test url == - "http://oasis.caiso.com/oasisapi/SingleZip?resultformat=6&queryname=PRC_LMP&version=1&startdatetime=20230104T07:00-0300&enddatetime=20230105T07:00-0000&market_run_id=DAM" + "http://oasis.caiso.com/oasisapi/SingleZip?resultformat=6&queryname=PRC_INTVL_LMP&version=1&startdatetime=20230104T07:00-0300&enddatetime=20230105T07:00-0000&market_run_id=DAM" end end From 1b9ffe055150349c359573ecdc428d14e6337838 Mon Sep 17 00:00:00 2001 From: Thiago Novaes Date: Wed, 30 Oct 2024 11:36:24 +0100 Subject: [PATCH 04/10] delete --- src/caiso/urls.jl | 25 ------------------------- test/caiso/urls.jl | 14 -------------- 2 files changed, 39 deletions(-) diff --git a/src/caiso/urls.jl b/src/caiso/urls.jl index b5a68eb..6fa91fd 100644 --- a/src/caiso/urls.jl +++ b/src/caiso/urls.jl @@ -26,28 +26,3 @@ function _get_caiso_price_url( )::AbstractString return _get_caiso_url("PRC_INTVL_LMP", startdate, stopdate, market_run_id) end - -""" - _get_caiso_real_time_url(startdate::AbstractString, stopdate::AbstractString)::AbstractString - -Returns the real time url for the Caiso API. -""" -function _get_caiso_real_time_url( - startdate::AbstractString, - stopdate::AbstractString, -)::AbstractString - return _get_caiso_price_url(startdate, stopdate, "RTM") -end - -""" - _get_caiso_day_ahead_url(startdate::AbstractString, stopdate::AbstractString)::AbstractString - -Returns the day ahead url for the Caiso API. -""" -function _get_caiso_day_ahead_url( - startdate::AbstractString, - stopdate::AbstractString, -)::AbstractString - return _get_caiso_price_url(startdate, stopdate, "DAM") -end - diff --git a/test/caiso/urls.jl b/test/caiso/urls.jl index 0617d9e..d6b2293 100644 --- a/test/caiso/urls.jl +++ b/test/caiso/urls.jl @@ -21,18 +21,4 @@ @test url == "http://oasis.caiso.com/oasisapi/SingleZip?resultformat=6&queryname=PRC_INTVL_LMP&version=1&startdatetime=20230104T07:00-0000&enddatetime=20230105T07:00-0000&market_run_id=RTM" end - @testset "_get_caiso_real_time_url" begin - startdate = "20230104T07:00-0300" - stopdate = "20230105T07:00-0300" - url = ElectricityMarketData._get_caiso_real_time_url(startdate, stopdate) - @test url == - "http://oasis.caiso.com/oasisapi/SingleZip?resultformat=6&queryname=PRC_INTVL_LMP&version=1&startdatetime=20230104T07:00-0300&enddatetime=20230105T07:00-0300&market_run_id=RTM" - end - @testset "_get_caiso_day_ahead_url" begin - startdate = "20230104T07:00-0300" - stopdate = "20230105T07:00-0000" - url = ElectricityMarketData._get_caiso_day_ahead_url(startdate, stopdate) - @test url == - "http://oasis.caiso.com/oasisapi/SingleZip?resultformat=6&queryname=PRC_INTVL_LMP&version=1&startdatetime=20230104T07:00-0300&enddatetime=20230105T07:00-0000&market_run_id=DAM" - end end From e94468ff7ac42926a640886dfbf172b2d8b38668 Mon Sep 17 00:00:00 2001 From: Thiago Novaes Date: Wed, 30 Oct 2024 13:00:09 +0100 Subject: [PATCH 05/10] raw --- src/ElectricityMarketData.jl | 3 + src/caiso/caiso_market.jl | 108 +++++++++++++++++++++++++++++++++++ src/caiso/utils.jl | 37 ++++++++++++ src/helpers/zip_helper.jl | 16 ++++++ 4 files changed, 164 insertions(+) create mode 100644 src/caiso/caiso_market.jl create mode 100644 src/caiso/utils.jl create mode 100644 src/helpers/zip_helper.jl diff --git a/src/ElectricityMarketData.jl b/src/ElectricityMarketData.jl index ff9fb33..53fb73f 100644 --- a/src/ElectricityMarketData.jl +++ b/src/ElectricityMarketData.jl @@ -25,6 +25,7 @@ export ElectricityMarket, # general include("helpers/http_helper.jl") +include("helpers/zip_helper.jl") include("electricity_market.jl") #miso @@ -38,7 +39,9 @@ include("pjm/utils.jl") include("pjm/parser.jl") #caiso +include("caiso/caiso_market.jl") include("caiso/datetime.jl") include("caiso/urls.jl") +include("caiso/utils.jl") end # module diff --git a/src/caiso/caiso_market.jl b/src/caiso/caiso_market.jl new file mode 100644 index 0000000..13a4d10 --- /dev/null +++ b/src/caiso/caiso_market.jl @@ -0,0 +1,108 @@ +""" + CaisoMarket + +struct representing the Midcontinent Independent System Operator (MISO). +""" + +Base.@kwdef struct CaisoMarket <: ElectricityMarket + url::String = "https://www.caiso.com/" + directory::String = "CaisoMarket" + timezone::TimeZone = tz"UTC-8" +end + +""" + available_time_series(::CaisoMarket) :: Vector{NamedTuple} + +Return Vector of `NamedTuple` of available time series for the given `market`. + +Ex: +``` +[ + (name="RT-load", unit="MW", resolution=Hour(1), first_date=DateTime("2021-01-01T00:00:00"), method=get_real_time_load_data, description="Real-time load data"), + (name="RT-LMP", unit="MWh", resolution=Hour(1), first_date=DateTime("2021-01-01T00:00:00"), method=get_real_time_lmp, description="Real-time Locational Marginal Price data"), +] +``` +""" +function available_time_series(::CaisoMarket)::Vector{NamedTuple} + return [ + ( + name = "RT-LMP", + unit = "\$/MWh", + resolution = Hour(1), + first_date = DateTime("2005-05-01T00:00:00"), + method = get_real_time_lmp, + description = "Real-time Locational Marginal Price data", + ), + ( + name = "DA-LMP", + unit = "\$/MWh", + resolution = Hour(1), + first_date = DateTime("2005-05-01T00:00:00"), + method = get_day_ahead_lmp, + description = "Day-ahead Locational Marginal Price data", + ), + ] +end + +""" + get_timezone(market::CaisoMarket) :: TimeZone + +Return the timezone of the given `market`. +""" +function get_timezone(market::CaisoMarket)::TimeZone + return market.timezone +end + +""" + get_real_time_lmp_raw_data(market::CaisoMarket, start_date::ZonedDateTime, end_date::ZonedDateTime; folder::AbstractString=tempdir()) + +Download raw data for Real-Time (RT) Locational Marginal Price (LMP) for the given `market` and `start_date` to `end_date` and save it in `folder`. +""" +function get_real_time_lmp_raw_data( + market::CaisoMarket, + start_date::ZonedDateTime, + end_date::ZonedDateTime; + folder::AbstractString = tempdir(), +)::Nothing + return _get_raw_data(market, "RTM", start_date, end_date, folder) +end + +""" + get_day_ahead_lmp_raw_data(market::CaisoMarket, start_date::ZonedDateTime, end_date::ZonedDateTime; folder::AbstractString=tempdir()) + +Download raw data for Day-Ahead (DA) Locational Marginal Price (LMP) for the given `market` and `start_date` to `end_date` and save it in `folder`. +""" +function get_day_ahead_lmp_raw_data( + market::CaisoMarket, + start_date::ZonedDateTime, + end_date::ZonedDateTime; + folder::AbstractString = tempdir(), +)::Nothing + return _get_raw_data(market, "DAM", start_date, end_date, folder) +end + +""" + get_real_time_lmp(market::CaisoMarket, start_date::ZonedDateTime, end_date::ZonedDateTime; folder::AbstractString=tempdir()) :: Tables.table + +Return a table with Real-Time (RT) Locational Marginal Price (LMP) data for the given `market` and `start_date` to `end_date`. +If the data is not available, download it and save it in `folder`. +""" +function get_real_time_lmp( + market::CaisoMarket, + start_date::ZonedDateTime, + end_date::ZonedDateTime; + folder::AbstractString = tempdir(), +) end + +""" + get_day_ahead_lmp(market::CaisoMarket, start_date::ZonedDateTime, end_date::ZonedDateTime; folder::AbstractString=tempdir()) :: Tables.table + +Return a table with Day-Ahead (DA) Locational Marginal Price (LMP) data for the given `market` and `start_date` to `end_date`. +If the data is not available, download it and save it in `folder`. +""" +function get_day_ahead_lmp( + market::CaisoMarket, + start_date::ZonedDateTime, + end_date::ZonedDateTime; + folder::AbstractString = tempdir(), +) end diff --git a/src/caiso/utils.jl b/src/caiso/utils.jl new file mode 100644 index 0000000..0a50200 --- /dev/null +++ b/src/caiso/utils.jl @@ -0,0 +1,37 @@ +""" + _get_raw_data(market::CaisoMarket, serie::AbstractString, start_date::ZonedDateTime, end_date::ZonedDateTime; folder::AbstractString)::Nothing + +Download the 'func' raw data for the given `market` and `start_date` to `end_date` and save it in `folder`. +""" +function _get_raw_data( + market::CaisoMarket, + serie::AbstractString, + start_date::ZonedDateTime, + end_date::ZonedDateTime, + folder::AbstractString, +)::Nothing + directory = mkpath(joinpath(folder, market.directory)) + tasks = Vector{Task}() + date = start_date + while date < end_date + start_formated = _get_zdt_formated(date) + next = date + Day(1) + end_formated = _get_zdt_formated(next) + url = _get_caiso_price_url(start_formated, end_formated, serie) + push!( + tasks, + _async_download( + url, + joinpath( + directory, + replace(start_formated, ":" => "") * "_" * serie * ".zip", + ), + ), + ) + date = next + end + for task in tasks + _unzip(fetch(task), x -> serie * "_" * x) + end + return nothing +end diff --git a/src/helpers/zip_helper.jl b/src/helpers/zip_helper.jl new file mode 100644 index 0000000..aec2f86 --- /dev/null +++ b/src/helpers/zip_helper.jl @@ -0,0 +1,16 @@ +""" + _unzip(filename::AbstractString, func::Function = x -> x)::Nothing + +Unzip all files of 'filename' adding to 'text'. +""" +function _unzip(filename::AbstractString, func::Function = x -> x)::Nothing + zip = ZipFile.Reader(filename) + for file in zip.files + dest_path = joinpath(dirname(filename), func(file.name)) + if !isfile(dest_path) + write(dest_path, read(file)) + end + end + close(zip) + return nothing +end From 02cfe6b29f2795d3870bf3f0ef8441c152d1dfcb Mon Sep 17 00:00:00 2001 From: Thiago Novaes Date: Wed, 30 Oct 2024 14:25:42 +0100 Subject: [PATCH 06/10] data --- src/caiso/caiso_market.jl | 8 +++-- src/caiso/utils.jl | 67 ++++++++++++++++++++++++++++++++++++--- src/helpers/zip_helper.jl | 8 +++-- 3 files changed, 74 insertions(+), 9 deletions(-) diff --git a/src/caiso/caiso_market.jl b/src/caiso/caiso_market.jl index 13a4d10..bc55f0d 100644 --- a/src/caiso/caiso_market.jl +++ b/src/caiso/caiso_market.jl @@ -92,7 +92,9 @@ function get_real_time_lmp( start_date::ZonedDateTime, end_date::ZonedDateTime; folder::AbstractString = tempdir(), -) end +) + return _get_data(market, "RTM", start_date, end_date, folder) +end """ get_day_ahead_lmp(market::CaisoMarket, start_date::ZonedDateTime, end_date::ZonedDateTime; folder::AbstractString=tempdir()) :: Tables.table @@ -105,4 +107,6 @@ function get_day_ahead_lmp( start_date::ZonedDateTime, end_date::ZonedDateTime; folder::AbstractString = tempdir(), -) end +) + return _get_data(market, "DAM", start_date, end_date, folder) +end diff --git a/src/caiso/utils.jl b/src/caiso/utils.jl index 0a50200..b3a6968 100644 --- a/src/caiso/utils.jl +++ b/src/caiso/utils.jl @@ -1,15 +1,15 @@ """ - _get_raw_data(market::CaisoMarket, serie::AbstractString, start_date::ZonedDateTime, end_date::ZonedDateTime; folder::AbstractString)::Nothing + _async_get_raw_data(market::CaisoMarket, serie::AbstractString, start_date::ZonedDateTime, end_date::ZonedDateTime; folder::AbstractString)::Vector{Task} Download the 'func' raw data for the given `market` and `start_date` to `end_date` and save it in `folder`. """ -function _get_raw_data( +function _async_get_raw_data( market::CaisoMarket, serie::AbstractString, start_date::ZonedDateTime, end_date::ZonedDateTime, folder::AbstractString, -)::Nothing +)::Vector{Task} directory = mkpath(joinpath(folder, market.directory)) tasks = Vector{Task}() date = start_date @@ -30,8 +30,67 @@ function _get_raw_data( ) date = next end + return tasks +end + +""" + _get_raw_data(market::CaisoMarket, serie::AbstractString, start_date::ZonedDateTime, end_date::ZonedDateTime; folder::AbstractString)::Nothing + +Download the 'func' raw data for the given `market` and `start_date` to `end_date` and save it in `folder`. +""" +function _get_raw_data( + market::CaisoMarket, + serie::AbstractString, + start_date::ZonedDateTime, + end_date::ZonedDateTime, + folder::AbstractString, +)::Nothing + tasks = _async_get_raw_data(market, serie, start_date, end_date, folder) for task in tasks - _unzip(fetch(task), x -> serie * "_" * x) + _ = _unzip(fetch(task), x -> serie * "_" * x) end return nothing end + +""" + _get_data(market::CaisoMarket, serie::AbstractString, start_date::ZonedDateTime, end_date::ZonedDateTime; folder::AbstractString) + +Download the 'func' raw data for the given `market` and `start_date` to `end_date` and save it in `folder`. +""" +function _get_data( + market::CaisoMarket, + serie::AbstractString, + start_date::ZonedDateTime, + end_date::ZonedDateTime, + folder::AbstractString, +) + tasks = _async_get_raw_data(market, serie, start_date, end_date, folder) + dfs = Vector{DataFrame}() + for task in tasks + for file_name in _unzip(fetch(task), x -> serie * "_" * x) + df = CSV.read(file_name, DataFrame) + push!( + dfs, + unstack( + df, + [:INTERVALSTARTTIME_GMT, :MARKET_RUN_ID, :NODE], + :LMP_TYPE, + :MW, + ), + ) + end + end + df = vcat(dfs...) + rename!( + df, + Dict( + :INTERVALSTARTTIME_GMT => :Time, + :MARKET_RUN_ID => :Market, + :NODE => :Node, + :MCC => :Congestion, + :MCE => :Energy, + :MCL => :Loss, + ), + ) + return df +end diff --git a/src/helpers/zip_helper.jl b/src/helpers/zip_helper.jl index aec2f86..98cdac1 100644 --- a/src/helpers/zip_helper.jl +++ b/src/helpers/zip_helper.jl @@ -1,16 +1,18 @@ """ - _unzip(filename::AbstractString, func::Function = x -> x)::Nothing + _unzip(filename::AbstractString, func::Function = x -> x)::AbstractString Unzip all files of 'filename' adding to 'text'. """ -function _unzip(filename::AbstractString, func::Function = x -> x)::Nothing +function _unzip(filename::AbstractString, func::Function = x -> x)::Vector{AbstractString} zip = ZipFile.Reader(filename) + dest_paths = Vector{AbstractString}() for file in zip.files dest_path = joinpath(dirname(filename), func(file.name)) + push!(dest_paths, dest_path) if !isfile(dest_path) write(dest_path, read(file)) end end close(zip) - return nothing + return dest_paths end From 6648cba204dd391ed55e001fab415bb5017c752e Mon Sep 17 00:00:00 2001 From: Thiago Novaes Date: Wed, 30 Oct 2024 14:30:59 +0100 Subject: [PATCH 07/10] read me --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 75bec32..2680637 100644 --- a/README.md +++ b/README.md @@ -7,8 +7,8 @@ | Data | PJM | MISO | CAISO | | ------------- | :---: | :---: | :---: | -| Real Time LMP | ❌ | ✔️ | ❌ | -| Day-ahead LMP | ✔️ | ✔️ | ❌ | +| Real Time LMP | ❌ | ✔️ | ✔️ | +| Day-ahead LMP | ✔️ | ✔️ | ✔️ | | Load | ❌ | ❌ | ❌ | Example of getting data from PJM From ecb9f30cd5643dc69096b9a0f64f17452eea733b Mon Sep 17 00:00:00 2001 From: Thiago Novaes Date: Wed, 30 Oct 2024 15:42:59 +0100 Subject: [PATCH 08/10] fix url --- src/caiso/urls.jl | 3 ++- src/caiso/utils.jl | 16 +++++++--------- test/caiso/urls.jl | 8 ++++++++ 3 files changed, 17 insertions(+), 10 deletions(-) diff --git a/src/caiso/urls.jl b/src/caiso/urls.jl index 6fa91fd..d4f4370 100644 --- a/src/caiso/urls.jl +++ b/src/caiso/urls.jl @@ -24,5 +24,6 @@ function _get_caiso_price_url( stopdate::AbstractString, market_run_id::AbstractString, )::AbstractString - return _get_caiso_url("PRC_INTVL_LMP", startdate, stopdate, market_run_id) + queryname = market_run_id == "RTM" ? "PRC_INTVL_LMP" : "PRC_LMP" + return _get_caiso_url(queryname, startdate, stopdate, market_run_id) end diff --git a/src/caiso/utils.jl b/src/caiso/utils.jl index b3a6968..8ef182b 100644 --- a/src/caiso/utils.jl +++ b/src/caiso/utils.jl @@ -67,18 +67,16 @@ function _get_data( tasks = _async_get_raw_data(market, serie, start_date, end_date, folder) dfs = Vector{DataFrame}() for task in tasks + temp = nothing for file_name in _unzip(fetch(task), x -> serie * "_" * x) df = CSV.read(file_name, DataFrame) - push!( - dfs, - unstack( - df, - [:INTERVALSTARTTIME_GMT, :MARKET_RUN_ID, :NODE], - :LMP_TYPE, - :MW, - ), - ) + pivot = + unstack(df, [:INTERVALSTARTTIME_GMT, :MARKET_RUN_ID, :NODE], :LMP_TYPE, :MW) + temp = + isnothing(temp) ? pivot : + outerjoin(temp, pivot, on = [:INTERVALSTARTTIME_GMT, :MARKET_RUN_ID, :NODE]) end + push!(dfs, temp) end df = vcat(dfs...) rename!( diff --git a/test/caiso/urls.jl b/test/caiso/urls.jl index d6b2293..5dfe06c 100644 --- a/test/caiso/urls.jl +++ b/test/caiso/urls.jl @@ -21,4 +21,12 @@ @test url == "http://oasis.caiso.com/oasisapi/SingleZip?resultformat=6&queryname=PRC_INTVL_LMP&version=1&startdatetime=20230104T07:00-0000&enddatetime=20230105T07:00-0000&market_run_id=RTM" end + @testset "_get_caiso_price_url" begin + startdate = "20230104T07:00-0000" + stopdate = "20230105T07:00-0000" + market_run_id = "DAM" + url = ElectricityMarketData._get_caiso_price_url(startdate, stopdate, market_run_id) + @test url == + "http://oasis.caiso.com/oasisapi/SingleZip?resultformat=6&queryname=PRC_LMP&version=1&startdatetime=20230104T07:00-0000&enddatetime=20230105T07:00-0000&market_run_id=DAM" + end end From e051288851bae1b3ca9999c4fb25d8f7958f75b6 Mon Sep 17 00:00:00 2001 From: Thiago Novaes Date: Wed, 30 Oct 2024 16:11:18 +0100 Subject: [PATCH 09/10] fix --- src/caiso/caiso_market.jl | 40 +++++++++++++++++++++++++++++++++++---- src/caiso/urls.jl | 22 +++++++++++++++------ src/caiso/utils.jl | 20 +++++++++++++------- test/caiso/urls.jl | 10 ++++------ 4 files changed, 69 insertions(+), 23 deletions(-) diff --git a/src/caiso/caiso_market.jl b/src/caiso/caiso_market.jl index bc55f0d..65826af 100644 --- a/src/caiso/caiso_market.jl +++ b/src/caiso/caiso_market.jl @@ -64,7 +64,15 @@ function get_real_time_lmp_raw_data( end_date::ZonedDateTime; folder::AbstractString = tempdir(), )::Nothing - return _get_raw_data(market, "RTM", start_date, end_date, folder) + return _get_raw_data( + market, + "RTM", + Hour(1), + _get_caiso_real_time_url, + start_date, + end_date, + folder, + ) end """ @@ -78,7 +86,15 @@ function get_day_ahead_lmp_raw_data( end_date::ZonedDateTime; folder::AbstractString = tempdir(), )::Nothing - return _get_raw_data(market, "DAM", start_date, end_date, folder) + return _get_raw_data( + market, + "DAM", + Day(1), + _get_caiso_day_ahead_url, + start_date, + end_date, + folder, + ) end """ @@ -93,7 +109,15 @@ function get_real_time_lmp( end_date::ZonedDateTime; folder::AbstractString = tempdir(), ) - return _get_data(market, "RTM", start_date, end_date, folder) + return _get_data( + market, + "RTM", + Hour(1), + _get_caiso_real_time_url, + start_date, + end_date, + folder, + ) end """ @@ -108,5 +132,13 @@ function get_day_ahead_lmp( end_date::ZonedDateTime; folder::AbstractString = tempdir(), ) - return _get_data(market, "DAM", start_date, end_date, folder) + return _get_data( + market, + "DAM", + Day(1), + _get_caiso_day_ahead_url, + start_date, + end_date, + folder, + ) end diff --git a/src/caiso/urls.jl b/src/caiso/urls.jl index d4f4370..3fdcf10 100644 --- a/src/caiso/urls.jl +++ b/src/caiso/urls.jl @@ -15,15 +15,25 @@ function _get_caiso_url( end """ - _get_caiso_price_url(startdate::AbstractString, stopdate::AbstractString, market_run_id::AbstractString)::AbstractString + _get_caiso_day_ahead_url(startdate::AbstractString, stopdate::AbstractString)::AbstractString -Returns the price url for the Caiso API. +Returns the day ahead url for the Caiso API. """ -function _get_caiso_price_url( +function _get_caiso_day_ahead_url( + startdate::AbstractString, + stopdate::AbstractString, +)::AbstractString + return _get_caiso_url("PRC_LMP", startdate, stopdate, "DAM") +end + +""" + _get_caiso_real_time_url(startdate::AbstractString, stopdate::AbstractString)::AbstractString + +Returns the real time url for the Caiso API. +""" +function _get_caiso_real_time_url( startdate::AbstractString, stopdate::AbstractString, - market_run_id::AbstractString, )::AbstractString - queryname = market_run_id == "RTM" ? "PRC_INTVL_LMP" : "PRC_LMP" - return _get_caiso_url(queryname, startdate, stopdate, market_run_id) + return _get_caiso_url("PRC_INTVL_LMP", startdate, stopdate, "RTM") end diff --git a/src/caiso/utils.jl b/src/caiso/utils.jl index 8ef182b..39392a1 100644 --- a/src/caiso/utils.jl +++ b/src/caiso/utils.jl @@ -1,11 +1,13 @@ """ - _async_get_raw_data(market::CaisoMarket, serie::AbstractString, start_date::ZonedDateTime, end_date::ZonedDateTime; folder::AbstractString)::Vector{Task} + _async_get_raw_data(market::CaisoMarket, serie::AbstractString, period::Period, func::Function, start_date::ZonedDateTime, end_date::ZonedDateTime; folder::AbstractString)::Vector{Task} Download the 'func' raw data for the given `market` and `start_date` to `end_date` and save it in `folder`. """ function _async_get_raw_data( market::CaisoMarket, serie::AbstractString, + period::Period, + func::Function, start_date::ZonedDateTime, end_date::ZonedDateTime, folder::AbstractString, @@ -15,9 +17,9 @@ function _async_get_raw_data( date = start_date while date < end_date start_formated = _get_zdt_formated(date) - next = date + Day(1) + next = date + period end_formated = _get_zdt_formated(next) - url = _get_caiso_price_url(start_formated, end_formated, serie) + url = func(start_formated, end_formated) push!( tasks, _async_download( @@ -34,18 +36,20 @@ function _async_get_raw_data( end """ - _get_raw_data(market::CaisoMarket, serie::AbstractString, start_date::ZonedDateTime, end_date::ZonedDateTime; folder::AbstractString)::Nothing + _get_raw_data(market::CaisoMarket, serie::AbstractString, period::Period, func::Function, start_date::ZonedDateTime, end_date::ZonedDateTime; folder::AbstractString)::Nothing Download the 'func' raw data for the given `market` and `start_date` to `end_date` and save it in `folder`. """ function _get_raw_data( market::CaisoMarket, serie::AbstractString, + period::Period, + func::Function, start_date::ZonedDateTime, end_date::ZonedDateTime, folder::AbstractString, )::Nothing - tasks = _async_get_raw_data(market, serie, start_date, end_date, folder) + tasks = _async_get_raw_data(market, serie, period, func, start_date, end_date, folder) for task in tasks _ = _unzip(fetch(task), x -> serie * "_" * x) end @@ -53,18 +57,20 @@ function _get_raw_data( end """ - _get_data(market::CaisoMarket, serie::AbstractString, start_date::ZonedDateTime, end_date::ZonedDateTime; folder::AbstractString) + _get_data(market::CaisoMarket, serie::AbstractString, period::Period, func::Function, start_date::ZonedDateTime, end_date::ZonedDateTime; folder::AbstractString) Download the 'func' raw data for the given `market` and `start_date` to `end_date` and save it in `folder`. """ function _get_data( market::CaisoMarket, serie::AbstractString, + period::Period, + func::Function, start_date::ZonedDateTime, end_date::ZonedDateTime, folder::AbstractString, ) - tasks = _async_get_raw_data(market, serie, start_date, end_date, folder) + tasks = _async_get_raw_data(market, serie, period, func, start_date, end_date, folder) dfs = Vector{DataFrame}() for task in tasks temp = nothing diff --git a/test/caiso/urls.jl b/test/caiso/urls.jl index 5dfe06c..8b4c746 100644 --- a/test/caiso/urls.jl +++ b/test/caiso/urls.jl @@ -13,19 +13,17 @@ @test url == "http://oasis.caiso.com/oasisapi/SingleZip?resultformat=6&queryname=PRC_INTVL_LMP&version=1&startdatetime=20210105T07:00-0000&enddatetime=20210106T07:00-0000&market_run_id=RTM" end - @testset "_get_caiso_price_url" begin + @testset "_get_caiso_real_time_url" begin startdate = "20230104T07:00-0000" stopdate = "20230105T07:00-0000" - market_run_id = "RTM" - url = ElectricityMarketData._get_caiso_price_url(startdate, stopdate, market_run_id) + url = ElectricityMarketData._get_caiso_real_time_url(startdate, stopdate) @test url == "http://oasis.caiso.com/oasisapi/SingleZip?resultformat=6&queryname=PRC_INTVL_LMP&version=1&startdatetime=20230104T07:00-0000&enddatetime=20230105T07:00-0000&market_run_id=RTM" end - @testset "_get_caiso_price_url" begin + @testset "_get_caiso_day_ahead_url" begin startdate = "20230104T07:00-0000" stopdate = "20230105T07:00-0000" - market_run_id = "DAM" - url = ElectricityMarketData._get_caiso_price_url(startdate, stopdate, market_run_id) + url = ElectricityMarketData._get_caiso_day_ahead_url(startdate, stopdate) @test url == "http://oasis.caiso.com/oasisapi/SingleZip?resultformat=6&queryname=PRC_LMP&version=1&startdatetime=20230104T07:00-0000&enddatetime=20230105T07:00-0000&market_run_id=DAM" end From 300fefdbfd101bd724c240ffadf65d71c769765a Mon Sep 17 00:00:00 2001 From: Thiago Novaes Date: Wed, 30 Oct 2024 20:04:17 +0100 Subject: [PATCH 10/10] unzip --- test/helpers/zip_helper.jl | 13 +++++++++++++ test/runtests.jl | 2 ++ 2 files changed, 15 insertions(+) create mode 100644 test/helpers/zip_helper.jl diff --git a/test/helpers/zip_helper.jl b/test/helpers/zip_helper.jl new file mode 100644 index 0000000..146c3cc --- /dev/null +++ b/test/helpers/zip_helper.jl @@ -0,0 +1,13 @@ +@testset "zip_helper.jl" begin + @testset "_unzip" begin + mktempdir() do tempdir + zip = ZipFile.Writer("temp.zip") + ZipFile.addfile(zip, "file1.txt") + ZipFile.addfile(zip, "file2.txt") + close(zip) + _ = ElectricityMarketData._unzip("temp.zip", x -> "test_" * x) + @test isfile("test_file1.txt") + @test isfile("test_file2.txt") + end + end +end diff --git a/test/runtests.jl b/test/runtests.jl index fe2d9d5..cfd1bf9 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -3,12 +3,14 @@ using Dates using TimeZones using Test using HTTP +using ZipFile using DataFrames using CSV import ElectricityMarketData: get_timezone # general +include("helpers/zip_helper.jl") include("helpers/http_helper.jl") include("electricity_market.jl") #miso