diff --git a/README.md b/README.md index 75bec32..2680637 100644 --- a/README.md +++ b/README.md @@ -7,8 +7,8 @@ | Data | PJM | MISO | CAISO | | ------------- | :---: | :---: | :---: | -| Real Time LMP | ❌ | ✔️ | ❌ | -| Day-ahead LMP | ✔️ | ✔️ | ❌ | +| Real Time LMP | ❌ | ✔️ | ✔️ | +| Day-ahead LMP | ✔️ | ✔️ | ✔️ | | Load | ❌ | ❌ | ❌ | Example of getting data from PJM diff --git a/src/ElectricityMarketData.jl b/src/ElectricityMarketData.jl index c303673..53fb73f 100644 --- a/src/ElectricityMarketData.jl +++ b/src/ElectricityMarketData.jl @@ -25,6 +25,7 @@ export ElectricityMarket, # general include("helpers/http_helper.jl") +include("helpers/zip_helper.jl") include("electricity_market.jl") #miso @@ -37,4 +38,10 @@ include("pjm/urls.jl") include("pjm/utils.jl") include("pjm/parser.jl") +#caiso +include("caiso/caiso_market.jl") +include("caiso/datetime.jl") +include("caiso/urls.jl") +include("caiso/utils.jl") + end # module diff --git a/src/caiso/caiso_market.jl b/src/caiso/caiso_market.jl new file mode 100644 index 0000000..65826af --- /dev/null +++ b/src/caiso/caiso_market.jl @@ -0,0 +1,144 @@ +""" + CaisoMarket + +struct representing the Midcontinent Independent System Operator (MISO). +""" + +Base.@kwdef struct CaisoMarket <: ElectricityMarket + url::String = "https://www.caiso.com/" + directory::String = "CaisoMarket" + timezone::TimeZone = tz"UTC-8" +end + +""" + available_time_series(::CaisoMarket) :: Vector{NamedTuple} + +Return Vector of `NamedTuple` of available time series for the given `market`. + +Ex: +``` +[ + (name="RT-load", unit="MW", resolution=Hour(1), first_date=DateTime("2021-01-01T00:00:00"), method=get_real_time_load_data, description="Real-time load data"), + (name="RT-LMP", unit="MWh", resolution=Hour(1), first_date=DateTime("2021-01-01T00:00:00"), method=get_real_time_lmp, description="Real-time Locational Marginal Price data"), +] +``` +""" +function available_time_series(::CaisoMarket)::Vector{NamedTuple} + return [ + ( + name = "RT-LMP", + unit = "\$/MWh", + resolution = Hour(1), + first_date = DateTime("2005-05-01T00:00:00"), + method = get_real_time_lmp, + description = "Real-time Locational Marginal Price data", + ), + ( + name = "DA-LMP", + unit = "\$/MWh", + resolution = Hour(1), + first_date = DateTime("2005-05-01T00:00:00"), + method = get_day_ahead_lmp, + description = "Day-ahead Locational Marginal Price data", + ), + ] +end + +""" + get_timezone(market::CaisoMarket) :: TimeZone + +Return the timezone of the given `market`. +""" +function get_timezone(market::CaisoMarket)::TimeZone + return market.timezone +end + +""" + get_real_time_lmp_raw_data(market::CaisoMarket, start_date::ZonedDateTime, end_date::ZonedDateTime; folder::AbstractString=tempdir()) + +Download raw data for Real-Time (RT) Locational Marginal Price (LMP) for the given `market` and `start_date` to `end_date` and save it in `folder`. +""" +function get_real_time_lmp_raw_data( + market::CaisoMarket, + start_date::ZonedDateTime, + end_date::ZonedDateTime; + folder::AbstractString = tempdir(), +)::Nothing + return _get_raw_data( + market, + "RTM", + Hour(1), + _get_caiso_real_time_url, + start_date, + end_date, + folder, + ) +end + +""" + get_day_ahead_lmp_raw_data(market::CaisoMarket, start_date::ZonedDateTime, end_date::ZonedDateTime; folder::AbstractString=tempdir()) + +Download raw data for Day-Ahead (DA) Locational Marginal Price (LMP) for the given `market` and `start_date` to `end_date` and save it in `folder`. +""" +function get_day_ahead_lmp_raw_data( + market::CaisoMarket, + start_date::ZonedDateTime, + end_date::ZonedDateTime; + folder::AbstractString = tempdir(), +)::Nothing + return _get_raw_data( + market, + "DAM", + Day(1), + _get_caiso_day_ahead_url, + start_date, + end_date, + folder, + ) +end + +""" + get_real_time_lmp(market::CaisoMarket, start_date::ZonedDateTime, end_date::ZonedDateTime; folder::AbstractString=tempdir()) :: Tables.table + +Return a table with Real-Time (RT) Locational Marginal Price (LMP) data for the given `market` and `start_date` to `end_date`. +If the data is not available, download it and save it in `folder`. +""" +function get_real_time_lmp( + market::CaisoMarket, + start_date::ZonedDateTime, + end_date::ZonedDateTime; + folder::AbstractString = tempdir(), +) + return _get_data( + market, + "RTM", + Hour(1), + _get_caiso_real_time_url, + start_date, + end_date, + folder, + ) +end + +""" + get_day_ahead_lmp(market::CaisoMarket, start_date::ZonedDateTime, end_date::ZonedDateTime; folder::AbstractString=tempdir()) :: Tables.table + +Return a table with Day-Ahead (DA) Locational Marginal Price (LMP) data for the given `market` and `start_date` to `end_date`. +If the data is not available, download it and save it in `folder`. +""" +function get_day_ahead_lmp( + market::CaisoMarket, + start_date::ZonedDateTime, + end_date::ZonedDateTime; + folder::AbstractString = tempdir(), +) + return _get_data( + market, + "DAM", + Day(1), + _get_caiso_day_ahead_url, + start_date, + end_date, + folder, + ) +end diff --git a/src/caiso/datetime.jl b/src/caiso/datetime.jl new file mode 100644 index 0000000..1437397 --- /dev/null +++ b/src/caiso/datetime.jl @@ -0,0 +1,11 @@ +""" + _get_zdt_formated(zdt::ZonedDateTime)::AbstractString + +Get the zdt formated string in the format "yyyymmddTHH:MMzzzz". +""" +function _get_zdt_formated(zdt::ZonedDateTime)::AbstractString + return string( + Dates.format(zdt, "yyyymmddTHH:MM"), + replace(Dates.format(zdt, "zzzz"), ":" => ""), + ) +end diff --git a/src/caiso/urls.jl b/src/caiso/urls.jl new file mode 100644 index 0000000..3fdcf10 --- /dev/null +++ b/src/caiso/urls.jl @@ -0,0 +1,39 @@ +""" + _get_caiso_url(queryname::AbstractString, startdate::AbstractString, stopdate::AbstractString, market_run_id::AbstractString, version::AbstractString = "1", resultformat::AbstractString="6")::AbstractString + +Returns the url for the Caiso API. +""" +function _get_caiso_url( + queryname::AbstractString, + startdate::AbstractString, + stopdate::AbstractString, + market_run_id::AbstractString, + version::AbstractString = "1", + resultformat::AbstractString = "6", +)::AbstractString + return "http://oasis.caiso.com/oasisapi/SingleZip?resultformat=$resultformat&queryname=$queryname&version=$version&startdatetime=$startdate&enddatetime=$stopdate&market_run_id=$market_run_id" +end + +""" + _get_caiso_day_ahead_url(startdate::AbstractString, stopdate::AbstractString)::AbstractString + +Returns the day ahead url for the Caiso API. +""" +function _get_caiso_day_ahead_url( + startdate::AbstractString, + stopdate::AbstractString, +)::AbstractString + return _get_caiso_url("PRC_LMP", startdate, stopdate, "DAM") +end + +""" + _get_caiso_real_time_url(startdate::AbstractString, stopdate::AbstractString)::AbstractString + +Returns the real time url for the Caiso API. +""" +function _get_caiso_real_time_url( + startdate::AbstractString, + stopdate::AbstractString, +)::AbstractString + return _get_caiso_url("PRC_INTVL_LMP", startdate, stopdate, "RTM") +end diff --git a/src/caiso/utils.jl b/src/caiso/utils.jl new file mode 100644 index 0000000..39392a1 --- /dev/null +++ b/src/caiso/utils.jl @@ -0,0 +1,100 @@ +""" + _async_get_raw_data(market::CaisoMarket, serie::AbstractString, period::Period, func::Function, start_date::ZonedDateTime, end_date::ZonedDateTime; folder::AbstractString)::Vector{Task} + +Download the 'func' raw data for the given `market` and `start_date` to `end_date` and save it in `folder`. +""" +function _async_get_raw_data( + market::CaisoMarket, + serie::AbstractString, + period::Period, + func::Function, + start_date::ZonedDateTime, + end_date::ZonedDateTime, + folder::AbstractString, +)::Vector{Task} + directory = mkpath(joinpath(folder, market.directory)) + tasks = Vector{Task}() + date = start_date + while date < end_date + start_formated = _get_zdt_formated(date) + next = date + period + end_formated = _get_zdt_formated(next) + url = func(start_formated, end_formated) + push!( + tasks, + _async_download( + url, + joinpath( + directory, + replace(start_formated, ":" => "") * "_" * serie * ".zip", + ), + ), + ) + date = next + end + return tasks +end + +""" + _get_raw_data(market::CaisoMarket, serie::AbstractString, period::Period, func::Function, start_date::ZonedDateTime, end_date::ZonedDateTime; folder::AbstractString)::Nothing + +Download the 'func' raw data for the given `market` and `start_date` to `end_date` and save it in `folder`. +""" +function _get_raw_data( + market::CaisoMarket, + serie::AbstractString, + period::Period, + func::Function, + start_date::ZonedDateTime, + end_date::ZonedDateTime, + folder::AbstractString, +)::Nothing + tasks = _async_get_raw_data(market, serie, period, func, start_date, end_date, folder) + for task in tasks + _ = _unzip(fetch(task), x -> serie * "_" * x) + end + return nothing +end + +""" + _get_data(market::CaisoMarket, serie::AbstractString, period::Period, func::Function, start_date::ZonedDateTime, end_date::ZonedDateTime; folder::AbstractString) + +Download the 'func' raw data for the given `market` and `start_date` to `end_date` and save it in `folder`. +""" +function _get_data( + market::CaisoMarket, + serie::AbstractString, + period::Period, + func::Function, + start_date::ZonedDateTime, + end_date::ZonedDateTime, + folder::AbstractString, +) + tasks = _async_get_raw_data(market, serie, period, func, start_date, end_date, folder) + dfs = Vector{DataFrame}() + for task in tasks + temp = nothing + for file_name in _unzip(fetch(task), x -> serie * "_" * x) + df = CSV.read(file_name, DataFrame) + pivot = + unstack(df, [:INTERVALSTARTTIME_GMT, :MARKET_RUN_ID, :NODE], :LMP_TYPE, :MW) + temp = + isnothing(temp) ? pivot : + outerjoin(temp, pivot, on = [:INTERVALSTARTTIME_GMT, :MARKET_RUN_ID, :NODE]) + end + push!(dfs, temp) + end + df = vcat(dfs...) + rename!( + df, + Dict( + :INTERVALSTARTTIME_GMT => :Time, + :MARKET_RUN_ID => :Market, + :NODE => :Node, + :MCC => :Congestion, + :MCE => :Energy, + :MCL => :Loss, + ), + ) + return df +end diff --git a/src/helpers/zip_helper.jl b/src/helpers/zip_helper.jl new file mode 100644 index 0000000..98cdac1 --- /dev/null +++ b/src/helpers/zip_helper.jl @@ -0,0 +1,18 @@ +""" + _unzip(filename::AbstractString, func::Function = x -> x)::AbstractString + +Unzip all files of 'filename' adding to 'text'. +""" +function _unzip(filename::AbstractString, func::Function = x -> x)::Vector{AbstractString} + zip = ZipFile.Reader(filename) + dest_paths = Vector{AbstractString}() + for file in zip.files + dest_path = joinpath(dirname(filename), func(file.name)) + push!(dest_paths, dest_path) + if !isfile(dest_path) + write(dest_path, read(file)) + end + end + close(zip) + return dest_paths +end diff --git a/test/caiso/datetime.jl b/test/caiso/datetime.jl new file mode 100644 index 0000000..1997838 --- /dev/null +++ b/test/caiso/datetime.jl @@ -0,0 +1,7 @@ +@testset "caiso/date.jl" begin + @testset "_get_zdt_formated" begin + zdt = ZonedDateTime(2023, 5, 4, 7, 2, 13, tz"UTC-3") + formated = ElectricityMarketData._get_zdt_formated(zdt) + @test formated == "20230504T07:02-0300" + end +end diff --git a/test/caiso/urls.jl b/test/caiso/urls.jl new file mode 100644 index 0000000..8b4c746 --- /dev/null +++ b/test/caiso/urls.jl @@ -0,0 +1,30 @@ +@testset "caiso/urls.jl" begin + @testset "_get_caiso_url" begin + queryname = "PRC_INTVL_LMP" + startdate = "20210105T07:00-0000" + stopdate = "20210106T07:00-0000" + market_run_id = "RTM" + url = ElectricityMarketData._get_caiso_url( + queryname, + startdate, + stopdate, + market_run_id, + ) + @test url == + "http://oasis.caiso.com/oasisapi/SingleZip?resultformat=6&queryname=PRC_INTVL_LMP&version=1&startdatetime=20210105T07:00-0000&enddatetime=20210106T07:00-0000&market_run_id=RTM" + end + @testset "_get_caiso_real_time_url" begin + startdate = "20230104T07:00-0000" + stopdate = "20230105T07:00-0000" + url = ElectricityMarketData._get_caiso_real_time_url(startdate, stopdate) + @test url == + "http://oasis.caiso.com/oasisapi/SingleZip?resultformat=6&queryname=PRC_INTVL_LMP&version=1&startdatetime=20230104T07:00-0000&enddatetime=20230105T07:00-0000&market_run_id=RTM" + end + @testset "_get_caiso_day_ahead_url" begin + startdate = "20230104T07:00-0000" + stopdate = "20230105T07:00-0000" + url = ElectricityMarketData._get_caiso_day_ahead_url(startdate, stopdate) + @test url == + "http://oasis.caiso.com/oasisapi/SingleZip?resultformat=6&queryname=PRC_LMP&version=1&startdatetime=20230104T07:00-0000&enddatetime=20230105T07:00-0000&market_run_id=DAM" + end +end diff --git a/test/helpers/zip_helper.jl b/test/helpers/zip_helper.jl new file mode 100644 index 0000000..146c3cc --- /dev/null +++ b/test/helpers/zip_helper.jl @@ -0,0 +1,13 @@ +@testset "zip_helper.jl" begin + @testset "_unzip" begin + mktempdir() do tempdir + zip = ZipFile.Writer("temp.zip") + ZipFile.addfile(zip, "file1.txt") + ZipFile.addfile(zip, "file2.txt") + close(zip) + _ = ElectricityMarketData._unzip("temp.zip", x -> "test_" * x) + @test isfile("test_file1.txt") + @test isfile("test_file2.txt") + end + end +end diff --git a/test/runtests.jl b/test/runtests.jl index 41553b1..cfd1bf9 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -3,12 +3,14 @@ using Dates using TimeZones using Test using HTTP +using ZipFile using DataFrames using CSV import ElectricityMarketData: get_timezone # general +include("helpers/zip_helper.jl") include("helpers/http_helper.jl") include("electricity_market.jl") #miso @@ -18,3 +20,6 @@ include("pjm/pjm_market.jl") include("pjm/urls.jl") include("pjm/utils.jl") include("pjm/parser.jl") +#caiso +include("caiso/datetime.jl") +include("caiso/urls.jl")