From 9d80c28e68fc34684a121b0b602d82e749fe221d Mon Sep 17 00:00:00 2001 From: Leland Boeman Date: Tue, 12 May 2020 14:26:48 -0700 Subject: [PATCH 01/25] remove non-reporting sites --- .../io/reference_observations/sfa_reference_sites.csv | 4 ---- 1 file changed, 4 deletions(-) diff --git a/solarforecastarbiter/io/reference_observations/sfa_reference_sites.csv b/solarforecastarbiter/io/reference_observations/sfa_reference_sites.csv index b13b498e2..a5986c7db 100644 --- a/solarforecastarbiter/io/reference_observations/sfa_reference_sites.csv +++ b/solarforecastarbiter/io/reference_observations/sfa_reference_sites.csv @@ -32,19 +32,15 @@ interval_length,name,latitude,longitude,elevation,network_api_id,network_api_abb 1,Salem OR,44.92100000000001,-123.01799999999999,62,94806.0,SA,Etc/GMT+8,"Peterson, J., and Vignola, F., 2017: Structure of a Comprehensive Solar Radiation Dataset. Proceedings of the ASES National Solar Conference 2017. doi: 10.18086/solar.2017.07.02",UO SRML 1,Portland OR PV,45.51,-122.69,70,94808.0,PS,Etc/GMT+8,"Peterson, J., and Vignola, F., 2017: Structure of a Comprehensive Solar Radiation Dataset. Proceedings of the ASES National Solar Conference 2017. doi: 10.18086/solar.2017.07.02",UO SRML 1,Cheney WA,47.49,-117.589,777,94158.0,CY,Etc/GMT+8,"Peterson, J., and Vignola, F., 2017: Structure of a Comprehensive Solar Radiation Dataset. Proceedings of the ASES National Solar Conference 2017. doi: 10.18086/solar.2017.07.02",UO SRML -1,Dillon MT,45.21,-112.64,1590,94145.0,DI,Etc/GMT+8,"Peterson, J., and Vignola, F., 2017: Structure of a Comprehensive Solar Radiation Dataset. Proceedings of the ASES National Solar Conference 2017. doi: 10.18086/solar.2017.07.02",UO SRML 1,Eugene OR,44.05,-123.07,150,94255.0,EU,Etc/GMT+8,"Peterson, J., and Vignola, F., 2017: Structure of a Comprehensive Solar Radiation Dataset. Proceedings of the ASES National Solar Conference 2017. doi: 10.18086/solar.2017.07.02",UO SRML 1,Burns OR,43.52,-119.02,1265,94170.0,BU,Etc/GMT+8,"Peterson, J., and Vignola, F., 2017: Structure of a Comprehensive Solar Radiation Dataset. Proceedings of the ASES National Solar Conference 2017. doi: 10.18086/solar.2017.07.02",UO SRML 5,Aberdeen ID,42.95,-112.78,1433,94174.0,AB,Etc/GMT+8,"Peterson, J., and Vignola, F., 2017: Structure of a Comprehensive Solar Radiation Dataset. Proceedings of the ASES National Solar Conference 2017. doi: 10.18086/solar.2017.07.02",UO SRML 1,Silver Lake OR,43.12,-121.06,1355,94249.0,SL,Etc/GMT+8,"Peterson, J., and Vignola, F., 2017: Structure of a Comprehensive Solar Radiation Dataset. Proceedings of the ASES National Solar Conference 2017. doi: 10.18086/solar.2017.07.02",UO SRML 1,Bend OR,44.06,-121.31,1124,94807.0,BD,Etc/GMT+8,"Peterson, J., and Vignola, F., 2017: Structure of a Comprehensive Solar Radiation Dataset. Proceedings of the ASES National Solar Conference 2017. doi: 10.18086/solar.2017.07.02",UO SRML -15,Moab UT,38.58,-109.54,1000,94102.0,MO,Etc/GMT+8,"Peterson, J., and Vignola, F., 2017: Structure of a Comprehensive Solar Radiation Dataset. Proceedings of the ASES National Solar Conference 2017. doi: 10.18086/solar.2017.07.02",UO SRML -15,Green River WY,41.46,-109.44,1000,94101.0,GR,Etc/GMT+8,"Peterson, J., and Vignola, F., 2017: Structure of a Comprehensive Solar Radiation Dataset. Proceedings of the ASES National Solar Conference 2017. doi: 10.18086/solar.2017.07.02",UO SRML 15,Seattle WA,47.68,-122.25,20,94290.0,SE,Etc/GMT+8,"Peterson, J., and Vignola, F., 2017: Structure of a Comprehensive Solar Radiation Dataset. Proceedings of the ASES National Solar Conference 2017. doi: 10.18086/solar.2017.07.02",UO SRML 15,Forest Grove OR,45.55,-123.08,55,94008.0,FG,Etc/GMT+8,"Peterson, J., and Vignola, F., 2017: Structure of a Comprehensive Solar Radiation Dataset. Proceedings of the ASES National Solar Conference 2017. doi: 10.18086/solar.2017.07.02",UO SRML 15,Picabo ID,43.31,-114.17,1472,94172.0,PI,Etc/GMT+8,"Peterson, J., and Vignola, F., 2017: Structure of a Comprehensive Solar Radiation Dataset. Proceedings of the ASES National Solar Conference 2017. doi: 10.18086/solar.2017.07.02",UO SRML 15,Twin Falls ID,42.55,-114.35,1200,94171.0,TW,Etc/GMT+8,"Peterson, J., and Vignola, F., 2017: Structure of a Comprehensive Solar Radiation Dataset. Proceedings of the ASES National Solar Conference 2017. doi: 10.18086/solar.2017.07.02",UO SRML -15,Boise ID,43.62,-116.21,701,94182.0,BO,Etc/GMT+8,"Peterson, J., and Vignola, F., 2017: Structure of a Comprehensive Solar Radiation Dataset. Proceedings of the ASES National Solar Conference 2017. doi: 10.18086/solar.2017.07.02",UO SRML 15,Parma ID,43.8,-116.94,678,94173.0,PA,Etc/GMT+8,"Peterson, J., and Vignola, F., 2017: Structure of a Comprehensive Solar Radiation Dataset. Proceedings of the ASES National Solar Conference 2017. doi: 10.18086/solar.2017.07.02",UO SRML 15,Christmas Valley OR,43.24,-120.88,1325,94251.0,CH,Etc/GMT+8,"Peterson, J., and Vignola, F., 2017: Structure of a Comprehensive Solar Radiation Dataset. Proceedings of the ASES National Solar Conference 2017. doi: 10.18086/solar.2017.07.02",UO SRML 15,Bend OR,43.99,-121.35,1100,94256.0,BE,Etc/GMT+8,"Peterson, J., and Vignola, F., 2017: Structure of a Comprehensive Solar Radiation Dataset. Proceedings of the ASES National Solar Conference 2017. doi: 10.18086/solar.2017.07.02",UO SRML From 4be09a7b0ebdd964ede335e8416af3c05835a96b Mon Sep 17 00:00:00 2001 From: Leland Boeman Date: Tue, 12 May 2020 14:27:04 -0700 Subject: [PATCH 02/25] temp commit for rebase --- .../io/reference_observations/srml.py | 113 ++++++++++++++---- 1 file changed, 93 insertions(+), 20 deletions(-) diff --git a/solarforecastarbiter/io/reference_observations/srml.py b/solarforecastarbiter/io/reference_observations/srml.py index b34f9b3ef..139935ec6 100644 --- a/solarforecastarbiter/io/reference_observations/srml.py +++ b/solarforecastarbiter/io/reference_observations/srml.py @@ -18,6 +18,26 @@ 'dhi_': 'dhi', 'wind_speed_': 'wind_speed', 'temp_air_': 'air_temperature', + '5': 'power', +} + +# SRMLs data element numbers use the second digit to indicate +# surface tilt, this map assists in looking up the correct data +# element number by the surface tilt of a station. +srml_surface_tilt_map = { + 15: 1, + 25: 2, + 30: 3, + 45: 5, + 60: 6, +} + +# SRMLs data element numbers use the third digit to indicate +# surface azimuth with a general direction e.g. south-facing +srml_azimuth_map = { + 270.0: 4, # west-facing + 180.0: 6, # south-facing + 90.0: 8, # east-facing } # maps SolarForecastArbiter interval_label to the SRML infix which @@ -36,6 +56,20 @@ logger = logging.getLogger('reference_data') +def _site_power_prefix(tilt, azimuth): + """ + Parameters + ---------- + tilt: float + Site surface tilt. + azimuth: float + Site surface azimuth. + + Returns + ------- + str + The prefix to use when filtering srml data columns. + """ def request_data(site, year, month): """Makes a request for each file type until successful or we run out of filetypes. @@ -80,6 +114,8 @@ def request_data(site, year, month): continue else: return srml_month + logger.warning(f'Could not retrieve data for site {site.name} on ' + f'{year}/{month}.') def fetch(api, site, start, end): @@ -154,8 +190,8 @@ def initialize_site_observations(api, site): labels to differentiate between measurements recorded by different instruments. """ - start = pd.Timestamp.now() - end = pd.Timestamp.now() + start = pd.Timestamp.now() - pd.Timedelta('30 days') + end = start try: extra_params = common.decode_extra_parameters(site) except ValueError: @@ -178,24 +214,61 @@ def initialize_site_observations(api, site): f'for SRML site {site_name}.') return for variable in srml_variable_map.keys(): - matches = [col for col in site_df.columns if variable in col] - for match in matches: - observation_extra_parameters = extra_params.copy() - observation_extra_parameters.update({ - 'network_data_label': match}) - try: - # Here, we pass a name with match instead of variable - # to differentiate between multiple observations of - # the same variable - common.create_observation( - api, site, srml_variable_map[variable], - name=f'{site_name} {match}', - interval_label='beginning', - extra_params=observation_extra_parameters) - except HTTPError as e: - logger.error( - f'Failed to create {variable} observation at Site ' - f'{site.name}. Error: {e.response.text}') + matches = [col for col in site_df.columns + if col.startswith(variable)] + if variable == '5': + # create an aggregated power observations + if hasattr(site, "modeling_parameters"): + site_power_prefix = _site_power_prefix( + site.modeling_parameters.surface_tilt, + site.modeling_parameters.surface_azimuth, + ) + # limit power to data with matching surface_tilt/azimuth + matches = [match for match in matches + if match.startswith(site_power_prefix)] + if matches: + # from matching raw data files against the + # comprehensive format, it appears that ac power + # measurements are suffixed with a letter, while + # dc is suffixed with a number. + dc_matches = [match for match in matches if isalpha(match[-1])] + ac_matches = [match for match in matches if isnumeric(match[-1])] + observation_extra_parameters = extra_params.copy() + observation_extra_parameters.update({ + 'network_data_label': ','.join(matches)}) + try: + # Here, we pass a name with match instead of + # variable to differentiate between multiple + # observations of the same variable + common.create_observation( + api, site, srml_variable_map[variable], + name=f'{site_name} {srml_variable_map[variable]}', + interval_label='beginning', + extra_params=observation_extra_parameters) + except HTTPError as e: + logger.error( + f'Failed to create {variable} observation at ' + f'Site {site.name}. Error: {e.response.text}') + else: + continue + else: + for match in matches: + observation_extra_parameters = extra_params.copy() + observation_extra_parameters.update({ + 'network_data_label': match}) + try: + # Here, we pass a name with match instead of variable + # to differentiate between multiple observations of + # the same variable + common.create_observation( + api, site, srml_variable_map[variable], + name=f'{site_name} {match}', + interval_label='beginning', + extra_params=observation_extra_parameters) + except HTTPError as e: + logger.error( + f'Failed to create {variable} observation at Site ' + f'{site.name}. Error: {e.response.text}') def initialize_site_forecasts(api, site): From 5990695d547b11edacbc845f899b2701f8043fa1 Mon Sep 17 00:00:00 2001 From: Leland Boeman Date: Tue, 12 May 2020 14:34:48 -0700 Subject: [PATCH 03/25] add reference obs json files to gitignore --- .gitignore | 1 + .../srml_reference_sites.json | 521 ++++++++++++++++++ 2 files changed, 522 insertions(+) create mode 100644 solarforecastarbiter/io/reference_observations/srml_reference_sites.json diff --git a/.gitignore b/.gitignore index 2666874d1..505108ace 100644 --- a/.gitignore +++ b/.gitignore @@ -112,6 +112,7 @@ venv.bak/ .vscode/ *.json +!solarforecastarbiter/io/reference_observations/*.json *.html !solarforecastarbiter/reports/templates/*.html *.md diff --git a/solarforecastarbiter/io/reference_observations/srml_reference_sites.json b/solarforecastarbiter/io/reference_observations/srml_reference_sites.json new file mode 100644 index 000000000..56e3a8d19 --- /dev/null +++ b/solarforecastarbiter/io/reference_observations/srml_reference_sites.json @@ -0,0 +1,521 @@ +{ + "observations": [ + { + "extra_parameters": "{\"network_data_label\": \"516A\", \"network\": \"UO SRML\", \"module\": \"Unisolar 68W/ Fronius IG 2000\"}", + "interval_label": "beginning", + "interval_length": 1.0, + "interval_value_type": "interval_mean", + "name": "Portland OR PV dc_power Unisolar", + "observation_id": "", + "provider": "", + "site": { + "elevation": 70.0, + "extra_parameters": "{\"network_api_id\": \"94808.0\", \"attribution\": \"Peterson, J., and Vignola, F., 2017: Structure of a Comprehensive Solar Radiation Dataset. Proceedings of the ASES National Solar Conference 2017. doi: 10.18086/solar.2017.07.02\", \"network\": \"UO SRML\", \"network_api_abbreviation\": \"PS\", \"observation_interval_length\": 1.0}", + "latitude": 45.51, + "longitude": -122.69, + "modeling_parameters": { + "ac_capacity": 0.004755, + "ac_loss_factor": 0.0, + "dc_capacity": 0.004755, + "dc_loss_factor": 0.0, + "surface_azimuth": 180.0, + "surface_tilt": 30.0, + "temperature_coefficient": 0.3, + "tracking_type": "fixed" + }, + "name": "Portland OR PV", + "provider": "", + "site_id": "" + }, + "uncertainty": 0, + "variable": "dc_power" + }, + { + "extra_parameters": "{\"network_data_label\": \"5161\", \"network\": \"UO SRML\", \"module\": \"Unisolar 68W/ Fronius IG 2000\"}", + "interval_label": "beginning", + "interval_length": 1.0, + "interval_value_type": "interval_mean", + "name": "Portland OR PV ac_power Unisolar", + "observation_id": "", + "provider": "", + "site": { + "elevation": 70.0, + "extra_parameters": "{\"network_api_id\": \"94808.0\", \"attribution\": \"Peterson, J., and Vignola, F., 2017: Structure of a Comprehensive Solar Radiation Dataset. Proceedings of the ASES National Solar Conference 2017. doi: 10.18086/solar.2017.07.02\", \"network\": \"UO SRML\", \"network_api_abbreviation\": \"PS\", \"observation_interval_length\": 1.0}", + "latitude": 45.51, + "longitude": -122.69, + "modeling_parameters": { + "ac_capacity": 0.004755, + "ac_loss_factor": 0.0, + "dc_capacity": 0.004755, + "dc_loss_factor": 0.0, + "surface_azimuth": 180.0, + "surface_tilt": 30.0, + "temperature_coefficient": 0.3, + "tracking_type": "fixed" + }, + "name": "Portland OR PV", + "provider": "", + "site_id": "" + }, + "uncertainty": 0, + "variable": "ac_power" + }, + { + "extra_parameters": "{\"network_data_label\": \"536B\", \"network\": \"UO SRML\", \"module\": \"Evergreen 190W/ Solectra PV1 1800\"}", + "interval_label": "beginning", + "interval_length": 1.0, + "interval_value_type": "interval_mean", + "name": "Portland OR PV dc_power Evergreen", + "observation_id": "", + "provider": "", + "site": { + "elevation": 70.0, + "extra_parameters": "{\"network_api_id\": \"94808.0\", \"attribution\": \"Peterson, J., and Vignola, F., 2017: Structure of a Comprehensive Solar Radiation Dataset. Proceedings of the ASES National Solar Conference 2017. doi: 10.18086/solar.2017.07.02\", \"network\": \"UO SRML\", \"network_api_abbreviation\": \"PS\", \"observation_interval_length\": 1.0}", + "latitude": 45.51, + "longitude": -122.69, + "modeling_parameters": { + "ac_capacity": 0.004755, + "ac_loss_factor": 0.0, + "dc_capacity": 0.004755, + "dc_loss_factor": 0.0, + "surface_azimuth": 180.0, + "surface_tilt": 30.0, + "temperature_coefficient": 0.3, + "tracking_type": "fixed" + }, + "name": "Portland OR PV", + "provider": "", + "site_id": "" + }, + "uncertainty": 0, + "variable": "dc_power" + }, + { + "extra_parameters": "{\"network_data_label\": \"536B\", \"network\": \"UO SRML\", \"module\": \"Evergreen 190W/ Solectra PV1 1800\"}", + "interval_label": "beginning", + "interval_length": 1.0, + "interval_value_type": "interval_mean", + "name": "Portland OR PV dc_power Evergreen", + "observation_id": "", + "provider": "", + "site": { + "elevation": 70.0, + "extra_parameters": "{\"network_api_id\": \"94808.0\", \"attribution\": \"Peterson, J., and Vignola, F., 2017: Structure of a Comprehensive Solar Radiation Dataset. Proceedings of the ASES National Solar Conference 2017. doi: 10.18086/solar.2017.07.02\", \"network\": \"UO SRML\", \"network_api_abbreviation\": \"PS\", \"observation_interval_length\": 1.0}", + "latitude": 45.51, + "longitude": -122.69, + "modeling_parameters": { + "ac_capacity": 0.004755, + "ac_loss_factor": 0.0, + "dc_capacity": 0.004755, + "dc_loss_factor": 0.0, + "surface_azimuth": 180.0, + "surface_tilt": 30.0, + "temperature_coefficient": 0.3, + "tracking_type": "fixed" + }, + "name": "Portland OR PV", + "provider": "", + "site_id": "" + }, + "uncertainty": 0, + "variable": "dc_power" + }, + { + "extra_parameters": "{\"network_data_label\": \"536C\", \"network\": \"UO SRML\", \"module\": \"Photowatt 105W/ PVP 1100\"}", + "interval_label": "beginning", + "interval_length": 1.0, + "interval_value_type": "interval_mean", + "name": "Portland OR PV dc_power Photowatt", + "observation_id": "", + "provider": "", + "site": { + "elevation": 70.0, + "extra_parameters": "{\"network_api_id\": \"94808.0\", \"attribution\": \"Peterson, J., and Vignola, F., 2017: Structure of a Comprehensive Solar Radiation Dataset. Proceedings of the ASES National Solar Conference 2017. doi: 10.18086/solar.2017.07.02\", \"network\": \"UO SRML\", \"network_api_abbreviation\": \"PS\", \"observation_interval_length\": 1.0}", + "latitude": 45.51, + "longitude": -122.69, + "modeling_parameters": { + "ac_capacity": 0.004755, + "ac_loss_factor": 0.0, + "dc_capacity": 0.004755, + "dc_loss_factor": 0.0, + "surface_azimuth": 180.0, + "surface_tilt": 30.0, + "temperature_coefficient": 0.3, + "tracking_type": "fixed" + }, + "name": "Portland OR PV", + "provider": "", + "site_id": "" + }, + "uncertainty": 0, + "variable": "dc_power" + }, + { + "extra_parameters": "{\"network_data_label\": \"536C\", \"network\": \"UO SRML\", \"module\": \"Photowatt 105W/ PVP 1100\"}", + "interval_label": "beginning", + "interval_length": 1.0, + "interval_value_type": "interval_mean", + "name": "Portland OR PV dc_power Photowatt", + "observation_id": "", + "provider": "", + "site": { + "elevation": 70.0, + "extra_parameters": "{\"network_api_id\": \"94808.0\", \"attribution\": \"Peterson, J., and Vignola, F., 2017: Structure of a Comprehensive Solar Radiation Dataset. Proceedings of the ASES National Solar Conference 2017. doi: 10.18086/solar.2017.07.02\", \"network\": \"UO SRML\", \"network_api_abbreviation\": \"PS\", \"observation_interval_length\": 1.0}", + "latitude": 45.51, + "longitude": -122.69, + "modeling_parameters": { + "ac_capacity": 0.004755, + "ac_loss_factor": 0.0, + "dc_capacity": 0.004755, + "dc_loss_factor": 0.0, + "surface_azimuth": 180.0, + "surface_tilt": 30.0, + "temperature_coefficient": 0.3, + "tracking_type": "fixed" + }, + "name": "Portland OR PV", + "provider": "", + "site_id": "" + }, + "uncertainty": 0, + "variable": "dc_power" + }, + { + "extra_parameters": "{\"network_data_label\": \"536D\", \"network\": \"UO SRML\", \"module\": \"Kaneka 60W/SMA 700\"}", + "interval_label": "beginning", + "interval_length": 1.0, + "interval_value_type": "interval_mean", + "name": "Portland OR PV dc_power Kaneka", + "observation_id": "", + "provider": "", + "site": { + "elevation": 70.0, + "extra_parameters": "{\"network_api_id\": \"94808.0\", \"attribution\": \"Peterson, J., and Vignola, F., 2017: Structure of a Comprehensive Solar Radiation Dataset. Proceedings of the ASES National Solar Conference 2017. doi: 10.18086/solar.2017.07.02\", \"network\": \"UO SRML\", \"network_api_abbreviation\": \"PS\", \"observation_interval_length\": 1.0}", + "latitude": 45.51, + "longitude": -122.69, + "modeling_parameters": { + "ac_capacity": 0.004755, + "ac_loss_factor": 0.0, + "dc_capacity": 0.004755, + "dc_loss_factor": 0.0, + "surface_azimuth": 180.0, + "surface_tilt": 30.0, + "temperature_coefficient": 0.3, + "tracking_type": "fixed" + }, + "name": "Portland OR PV", + "provider": "", + "site_id": "" + }, + "uncertainty": 0, + "variable": "dc_power" + }, + { + "extra_parameters": "{\"network_data_label\": \"536D\", \"network\": \"UO SRML\", \"module\": \"Kaneka 60W/SMA 700\"}", + "interval_label": "beginning", + "interval_length": 1.0, + "interval_value_type": "interval_mean", + "name": "Portland OR PV dc_power Kaneka", + "observation_id": "", + "provider": "", + "site": { + "elevation": 70.0, + "extra_parameters": "{\"network_api_id\": \"94808.0\", \"attribution\": \"Peterson, J., and Vignola, F., 2017: Structure of a Comprehensive Solar Radiation Dataset. Proceedings of the ASES National Solar Conference 2017. doi: 10.18086/solar.2017.07.02\", \"network\": \"UO SRML\", \"network_api_abbreviation\": \"PS\", \"observation_interval_length\": 1.0}", + "latitude": 45.51, + "longitude": -122.69, + "modeling_parameters": { + "ac_capacity": 0.004755, + "ac_loss_factor": 0.0, + "dc_capacity": 0.004755, + "dc_loss_factor": 0.0, + "surface_azimuth": 180.0, + "surface_tilt": 30.0, + "temperature_coefficient": 0.3, + "tracking_type": "fixed" + }, + "name": "Portland OR PV", + "provider": "", + "site_id": "" + }, + "uncertainty": 0, + "variable": "dc_power" + }, + { + "extra_parameters": "{\"network_data_label\": \"5161\", \"network\": \"UO SRML\", \"module\": \"\"}", + "interval_label": "beginning", + "interval_length": 1.0, + "interval_value_type": "interval_mean", + "name": "Ashland OR dc_power ", + "observation_id": "", + "provider": "", + "site": { + "elevation": 595.0, + "extra_parameters": "{\"network_api_id\": \"94040.0\", \"attribution\": \"Peterson, J., and Vignola, F., 2017: Structure of a Comprehensive Solar Radiation Dataset. Proceedings of the ASES National Solar Conference 2017. doi: 10.18086/solar.2017.07.02\", \"network\": \"UO SRML\", \"network_api_abbreviation\": \"AS\", \"observation_interval_length\": 1.0}", + "latitude": 42.19, + "longitude": -122.7, + "modeling_parameters": { + "ac_capacity": 0.02, + "ac_loss_factor": 0.0, + "dc_capacity": 0.02, + "dc_loss_factor": 0.0, + "surface_azimuth": 180.0, + "surface_tilt": 15.0, + "temperature_coefficient": 0.3, + "tracking_type": "fixed" + }, + "name": "Ashland OR", + "provider": "", + "site_id": "" + }, + "uncertainty": 0, + "variable": "dc_power" + }, + { + "extra_parameters": "{\"network_data_label\": \"5162\", \"network\": \"UO SRML\", \"module\": \"\"}", + "interval_label": "beginning", + "interval_length": 1.0, + "interval_value_type": "interval_mean", + "name": "Ashland OR dc_power ", + "observation_id": "", + "provider": "", + "site": { + "elevation": 595.0, + "extra_parameters": "{\"network_api_id\": \"94040.0\", \"attribution\": \"Peterson, J., and Vignola, F., 2017: Structure of a Comprehensive Solar Radiation Dataset. Proceedings of the ASES National Solar Conference 2017. doi: 10.18086/solar.2017.07.02\", \"network\": \"UO SRML\", \"network_api_abbreviation\": \"AS\", \"observation_interval_length\": 1.0}", + "latitude": 42.19, + "longitude": -122.7, + "modeling_parameters": { + "ac_capacity": 0.02, + "ac_loss_factor": 0.0, + "dc_capacity": 0.02, + "dc_loss_factor": 0.0, + "surface_azimuth": 180.0, + "surface_tilt": 15.0, + "temperature_coefficient": 0.3, + "tracking_type": "fixed" + }, + "name": "Ashland OR", + "provider": "", + "site_id": "" + }, + "uncertainty": 0, + "variable": "dc_power" + }, + { + "extra_parameters": "{\"network_data_label\": \"5361\", \"network\": \"UO SRML\", \"module\": \"Sharp 175 1050W\"}", + "interval_label": "beginning", + "interval_length": 1.0, + "interval_value_type": "interval_mean", + "name": "Kalapuya High School OR dc_power Sharp", + "observation_id": "", + "provider": "", + "site": { + "elevation": 0.0, + "extra_parameters": "{\"network_api_id\": \"94259.0\", \"attribution\": \"Peterson, J., and Vignola, F., 2017: Structure of a Comprehensive Solar Radiation Dataset. Proceedings of the ASES National Solar Conference 2017. doi: 10.18086/solar.2017.07.02\", \"network\": \"UO SRML\", \"network_api_abbreviation\": \"KA\", \"observation_interval_length\": 1.0}", + "latitude": 45.0, + "longitude": -120.0, + "modeling_parameters": { + "ac_capacity": 0.00105, + "ac_loss_factor": 0.0, + "dc_capacity": 0.00105, + "dc_loss_factor": 0.0, + "surface_azimuth": 180.0, + "surface_tilt": 30.0, + "temperature_coefficient": 0.3, + "tracking_type": "fixed" + }, + "name": "Kalapuya High School OR", + "provider": "", + "site_id": "" + }, + "uncertainty": 0, + "variable": "dc_power" + }, + { + "extra_parameters": "{\"network_data_label\": \"536A\", \"network\": \"UO SRML\", \"module\": \"Sharp 175 1050W\"}", + "interval_label": "beginning", + "interval_length": 1.0, + "interval_value_type": "interval_mean", + "name": "Kalapuya High School OR ac_power Sharp", + "observation_id": "", + "provider": "", + "site": { + "elevation": 0.0, + "extra_parameters": "{\"network_api_id\": \"94259.0\", \"attribution\": \"Peterson, J., and Vignola, F., 2017: Structure of a Comprehensive Solar Radiation Dataset. Proceedings of the ASES National Solar Conference 2017. doi: 10.18086/solar.2017.07.02\", \"network\": \"UO SRML\", \"network_api_abbreviation\": \"KA\", \"observation_interval_length\": 1.0}", + "latitude": 45.0, + "longitude": -120.0, + "modeling_parameters": { + "ac_capacity": 0.00105, + "ac_loss_factor": 0.0, + "dc_capacity": 0.00105, + "dc_loss_factor": 0.0, + "surface_azimuth": 180.0, + "surface_tilt": 30.0, + "temperature_coefficient": 0.3, + "tracking_type": "fixed" + }, + "name": "Kalapuya High School OR", + "provider": "", + "site_id": "" + }, + "uncertainty": 0, + "variable": "ac_power" + }, + { + "extra_parameters": "{\"network_data_label\": \"5361\", \"network\": \"UO SRML\", \"module\": \"Sunpower 255, inverter: SPR-3300x 240 VAC Sunpower\"}", + "interval_label": "beginning", + "interval_length": 1.0, + "interval_value_type": "interval_mean", + "name": "Bend OR (PV) ac_power Sunpower", + "observation_id": "", + "provider": "", + "site": { + "elevation": 1124.0, + "extra_parameters": "{\"network_api_id\": \"94807.0\", \"attribution\": \"Peterson, J., and Vignola, F., 2017: Structure of a Comprehensive Solar Radiation Dataset. Proceedings of the ASES National Solar Conference 2017. doi: 10.18086/solar.2017.07.02\", \"network\": \"UO SRML\", \"network_api_abbreviation\": \"BD\", \"observation_interval_length\": 1.0}", + "latitude": 44.06, + "longitude": -121.31, + "modeling_parameters": { + "ac_capacity": 0.0036, + "ac_loss_factor": 0.0, + "dc_capacity": 0.0036, + "dc_loss_factor": 0.0, + "surface_azimuth": 180.0, + "surface_tilt": 30.0, + "temperature_coefficient": 0.3, + "tracking_type": "fixed" + }, + "name": "Bend OR (PV)", + "provider": "", + "site_id": "" + }, + "uncertainty": 0, + "variable": "ac_power" + }, + { + "extra_parameters": "{\"network_data_label\": \"5369\", \"network\": \"UO SRML\", \"module\": \"Sunpower 255, inverter: SPR-3300x 240 VAC Sunpower\"}", + "interval_label": "beginning", + "interval_length": 1.0, + "interval_value_type": "interval_mean", + "name": "Bend OR (PV) dc_power Sunpower", + "observation_id": "", + "provider": "", + "site": { + "elevation": 1124.0, + "extra_parameters": "{\"network_api_id\": \"94807.0\", \"attribution\": \"Peterson, J., and Vignola, F., 2017: Structure of a Comprehensive Solar Radiation Dataset. Proceedings of the ASES National Solar Conference 2017. doi: 10.18086/solar.2017.07.02\", \"network\": \"UO SRML\", \"network_api_abbreviation\": \"BD\", \"observation_interval_length\": 1.0}", + "latitude": 44.06, + "longitude": -121.31, + "modeling_parameters": { + "ac_capacity": 0.0036, + "ac_loss_factor": 0.0, + "dc_capacity": 0.0036, + "dc_loss_factor": 0.0, + "surface_azimuth": 180.0, + "surface_tilt": 30.0, + "temperature_coefficient": 0.3, + "tracking_type": "fixed" + }, + "name": "Bend OR (PV)", + "provider": "", + "site_id": "" + }, + "uncertainty": 0, + "variable": "dc_power" + } + ], + "sites": [ + { + "elevation": 595.0, + "extra_parameters": "{\"network_api_id\": \"94040.0\", \"attribution\": \"Peterson, J., and Vignola, F., 2017: Structure of a Comprehensive Solar Radiation Dataset. Proceedings of the ASES National Solar Conference 2017. doi: 10.18086/solar.2017.07.02\", \"network\": \"UO SRML\", \"network_api_abbreviation\": \"AS\", \"observation_interval_length\": 1.0}", + "latitude": 42.19, + "longitude": -122.7, + "modeling_parameters": { + "ac_capacity": 0.02, + "ac_loss_factor": 0.0, + "dc_capacity": 0.02, + "dc_loss_factor": 0.0, + "surface_azimuth": 180.0, + "surface_tilt": 15.0, + "temperature_coefficient": 0.3, + "tracking_type": "fixed" + }, + "name": "Ashland OR", + "provider": "", + "site_id": "" + }, + { + "elevation": 0.0, + "extra_parameters": "{\"network_api_id\": \"94259.0\", \"attribution\": \"Peterson, J., and Vignola, F., 2017: Structure of a Comprehensive Solar Radiation Dataset. Proceedings of the ASES National Solar Conference 2017. doi: 10.18086/solar.2017.07.02\", \"network\": \"UO SRML\", \"network_api_abbreviation\": \"KA\", \"observation_interval_length\": 1.0}", + "latitude": 45.0, + "longitude": -120.0, + "modeling_parameters": { + "ac_capacity": 0.00105, + "ac_loss_factor": 0.0, + "dc_capacity": 0.00105, + "dc_loss_factor": 0.0, + "surface_azimuth": 180.0, + "surface_tilt": 30.0, + "temperature_coefficient": 0.3, + "tracking_type": "fixed" + }, + "name": "Kalapuya High School OR", + "provider": "", + "site_id": "" + }, + { + "elevation": 70.0, + "extra_parameters": "{\"network_api_id\": \"94808.0\", \"attribution\": \"Peterson, J., and Vignola, F., 2017: Structure of a Comprehensive Solar Radiation Dataset. Proceedings of the ASES National Solar Conference 2017. doi: 10.18086/solar.2017.07.02\", \"network\": \"UO SRML\", \"network_api_abbreviation\": \"PS\", \"observation_interval_length\": 1.0}", + "latitude": 45.51, + "longitude": -122.69, + "modeling_parameters": { + "ac_capacity": 0.004755, + "ac_loss_factor": 0.0, + "dc_capacity": 0.004755, + "dc_loss_factor": 0.0, + "surface_azimuth": 180.0, + "surface_tilt": 30.0, + "temperature_coefficient": 0.3, + "tracking_type": "fixed" + }, + "name": "Portland OR PV", + "provider": "", + "site_id": "" + }, + { + "elevation": 70.0, + "extra_parameters": "{\"network_api_id\": \"94808.0\", \"attribution\": \"Peterson, J., and Vignola, F., 2017: Structure of a Comprehensive Solar Radiation Dataset. Proceedings of the ASES National Solar Conference 2017. doi: 10.18086/solar.2017.07.02\", \"network\": \"UO SRML\", \"network_api_abbreviation\": \"PS\", \"observation_interval_length\": 1.0}", + "latitude": 45.51, + "longitude": -122.69, + "modeling_parameters": { + "ac_capacity": 0.004755, + "ac_loss_factor": 0.0, + "dc_capacity": 0.004755, + "dc_loss_factor": 0.0, + "surface_azimuth": 180.0, + "surface_tilt": 30.0, + "temperature_coefficient": 0.3, + "tracking_type": "fixed" + }, + "name": "Portland OR PV", + "provider": "", + "site_id": "" + }, + { + "elevation": 1124.0, + "extra_parameters": "{\"network_api_id\": \"94807.0\", \"attribution\": \"Peterson, J., and Vignola, F., 2017: Structure of a Comprehensive Solar Radiation Dataset. Proceedings of the ASES National Solar Conference 2017. doi: 10.18086/solar.2017.07.02\", \"network\": \"UO SRML\", \"network_api_abbreviation\": \"BD\", \"observation_interval_length\": 1.0}", + "latitude": 44.06, + "longitude": -121.31, + "modeling_parameters": { + "ac_capacity": 0.0036, + "ac_loss_factor": 0.0, + "dc_capacity": 0.0036, + "dc_loss_factor": 0.0, + "surface_azimuth": 180.0, + "surface_tilt": 30.0, + "temperature_coefficient": 0.3, + "tracking_type": "fixed" + }, + "name": "Bend OR (PV)", + "provider": "", + "site_id": "" + } + ] +} \ No newline at end of file From 5098c741f27615485ccad818f5f3fa397266596c Mon Sep 17 00:00:00 2001 From: Leland Boeman Date: Tue, 12 May 2020 17:17:18 -0700 Subject: [PATCH 04/25] declare pv srml sites and obs in json --- .../sfa_reference_sites.csv | 6 +- .../io/reference_observations/srml.py | 139 ++++++++---------- .../srml_reference_sites.json | 56 +++---- 3 files changed, 90 insertions(+), 111 deletions(-) diff --git a/solarforecastarbiter/io/reference_observations/sfa_reference_sites.csv b/solarforecastarbiter/io/reference_observations/sfa_reference_sites.csv index a5986c7db..3895bdfa0 100644 --- a/solarforecastarbiter/io/reference_observations/sfa_reference_sites.csv +++ b/solarforecastarbiter/io/reference_observations/sfa_reference_sites.csv @@ -26,9 +26,9 @@ # interval_length,name,latitude,longitude,elevation,network_api_id,network_api_abbreviation,timezone,attribution,network 1,Seattle UW WA,47.653999999999996,-122.309,70,94291.0,ST,Etc/GMT+5,"Peterson, J., and Vignola, F., 2017: Structure of a Comprehensive Solar Radiation Dataset. Proceedings of the ASES National Solar Conference 2017. doi: 10.18086/solar.2017.07.02",UO SRML -1,Ashland OR,42.19,-122.7,595,94040.0,AS,Etc/GMT+8,"Peterson, J., and Vignola, F., 2017: Structure of a Comprehensive Solar Radiation Dataset. Proceedings of the ASES National Solar Conference 2017. doi: 10.18086/solar.2017.07.02",UO SRML +1,Ashland OR PV,42.19,-122.7,595,94040.0,AS,Etc/GMT+8,"Peterson, J., and Vignola, F., 2017: Structure of a Comprehensive Solar Radiation Dataset. Proceedings of the ASES National Solar Conference 2017. doi: 10.18086/solar.2017.07.02",UO SRML 1,UO Solar Awning Eugene OR,44.05,-123.07,150,94255.0,AW,Etc/GMT+8,"Peterson, J., and Vignola, F., 2017: Structure of a Comprehensive Solar Radiation Dataset. Proceedings of the ASES National Solar Conference 2017. doi: 10.18086/solar.2017.07.02",UO SRML -1,Kalapuya High School OR,45.0,-120.0,0,94259.0,KA,Etc/GMT+8,"Peterson, J., and Vignola, F., 2017: Structure of a Comprehensive Solar Radiation Dataset. Proceedings of the ASES National Solar Conference 2017. doi: 10.18086/solar.2017.07.02",UO SRML +1,Kalapuya High School OR PV,45.0,-120.0,0,94259.0,KA,Etc/GMT+8,"Peterson, J., and Vignola, F., 2017: Structure of a Comprehensive Solar Radiation Dataset. Proceedings of the ASES National Solar Conference 2017. doi: 10.18086/solar.2017.07.02",UO SRML 1,Salem OR,44.92100000000001,-123.01799999999999,62,94806.0,SA,Etc/GMT+8,"Peterson, J., and Vignola, F., 2017: Structure of a Comprehensive Solar Radiation Dataset. Proceedings of the ASES National Solar Conference 2017. doi: 10.18086/solar.2017.07.02",UO SRML 1,Portland OR PV,45.51,-122.69,70,94808.0,PS,Etc/GMT+8,"Peterson, J., and Vignola, F., 2017: Structure of a Comprehensive Solar Radiation Dataset. Proceedings of the ASES National Solar Conference 2017. doi: 10.18086/solar.2017.07.02",UO SRML 1,Cheney WA,47.49,-117.589,777,94158.0,CY,Etc/GMT+8,"Peterson, J., and Vignola, F., 2017: Structure of a Comprehensive Solar Radiation Dataset. Proceedings of the ASES National Solar Conference 2017. doi: 10.18086/solar.2017.07.02",UO SRML @@ -36,7 +36,7 @@ interval_length,name,latitude,longitude,elevation,network_api_id,network_api_abb 1,Burns OR,43.52,-119.02,1265,94170.0,BU,Etc/GMT+8,"Peterson, J., and Vignola, F., 2017: Structure of a Comprehensive Solar Radiation Dataset. Proceedings of the ASES National Solar Conference 2017. doi: 10.18086/solar.2017.07.02",UO SRML 5,Aberdeen ID,42.95,-112.78,1433,94174.0,AB,Etc/GMT+8,"Peterson, J., and Vignola, F., 2017: Structure of a Comprehensive Solar Radiation Dataset. Proceedings of the ASES National Solar Conference 2017. doi: 10.18086/solar.2017.07.02",UO SRML 1,Silver Lake OR,43.12,-121.06,1355,94249.0,SL,Etc/GMT+8,"Peterson, J., and Vignola, F., 2017: Structure of a Comprehensive Solar Radiation Dataset. Proceedings of the ASES National Solar Conference 2017. doi: 10.18086/solar.2017.07.02",UO SRML -1,Bend OR,44.06,-121.31,1124,94807.0,BD,Etc/GMT+8,"Peterson, J., and Vignola, F., 2017: Structure of a Comprehensive Solar Radiation Dataset. Proceedings of the ASES National Solar Conference 2017. doi: 10.18086/solar.2017.07.02",UO SRML +1,Bend OR PV,44.06,-121.31,1124,94807.0,BD,Etc/GMT+8,"Peterson, J., and Vignola, F., 2017: Structure of a Comprehensive Solar Radiation Dataset. Proceedings of the ASES National Solar Conference 2017. doi: 10.18086/solar.2017.07.02",UO SRML 15,Seattle WA,47.68,-122.25,20,94290.0,SE,Etc/GMT+8,"Peterson, J., and Vignola, F., 2017: Structure of a Comprehensive Solar Radiation Dataset. Proceedings of the ASES National Solar Conference 2017. doi: 10.18086/solar.2017.07.02",UO SRML 15,Forest Grove OR,45.55,-123.08,55,94008.0,FG,Etc/GMT+8,"Peterson, J., and Vignola, F., 2017: Structure of a Comprehensive Solar Radiation Dataset. Proceedings of the ASES National Solar Conference 2017. doi: 10.18086/solar.2017.07.02",UO SRML 15,Picabo ID,43.31,-114.17,1472,94172.0,PI,Etc/GMT+8,"Peterson, J., and Vignola, F., 2017: Structure of a Comprehensive Solar Radiation Dataset. Proceedings of the ASES National Solar Conference 2017. doi: 10.18086/solar.2017.07.02",UO SRML diff --git a/solarforecastarbiter/io/reference_observations/srml.py b/solarforecastarbiter/io/reference_observations/srml.py index 139935ec6..4ca2dcb9d 100644 --- a/solarforecastarbiter/io/reference_observations/srml.py +++ b/solarforecastarbiter/io/reference_observations/srml.py @@ -1,5 +1,7 @@ import logging +import json from urllib import error +from pkg_resources import resource_filename, Requirement import pandas as pd @@ -7,10 +9,17 @@ from requests.exceptions import HTTPError +from solarforecastarbiter.datamodel import Observation from solarforecastarbiter.io.reference_observations import ( common, default_forecasts) +DEFAULT_SITEFILE = resource_filename( + Requirement.parse('solarforecastarbiter'), + 'solarforecastarbiter/io/reference_observations/' + 'srml_reference_sites.json') + + # maps the desired variable names to those returned by pvlib.iotools srml_variable_map = { 'ghi_': 'ghi', @@ -18,27 +27,8 @@ 'dhi_': 'dhi', 'wind_speed_': 'wind_speed', 'temp_air_': 'air_temperature', - '5': 'power', } -# SRMLs data element numbers use the second digit to indicate -# surface tilt, this map assists in looking up the correct data -# element number by the surface tilt of a station. -srml_surface_tilt_map = { - 15: 1, - 25: 2, - 30: 3, - 45: 5, - 60: 6, -} - -# SRMLs data element numbers use the third digit to indicate -# surface azimuth with a general direction e.g. south-facing -srml_azimuth_map = { - 270.0: 4, # west-facing - 180.0: 6, # south-facing - 90.0: 8, # east-facing -} # maps SolarForecastArbiter interval_label to the SRML infix which # designates the time resolution of each file. The list of file types @@ -56,20 +46,31 @@ logger = logging.getLogger('reference_data') -def _site_power_prefix(tilt, azimuth): - """ +def adjust_site_parameters(site): + """Inserts modeling parameters for sites with pv measurments + Parameters ---------- - tilt: float - Site surface tilt. - azimuth: float - Site surface azimuth. + site: dict Returns ------- - str - The prefix to use when filtering srml data columns. + dict + Copy of inputs plus a new key 'modeling_parameters'. """ + with open(DEFAULT_SITEFILE) as fp: + sites_metadata = json.load(fp)['sites'] + site_api_id = site['extra_parameters']['network_api_id'] + for site_metadata in sites_metadata: + site_extra_params = json.loads(site_metadata['extra_parameters']) + if site_extra_params['network_api_id'] == site_api_id: + site_out = site.copy() + site_out['modeling_parameters'] = site_metadata[ + 'modeling_parameters'] + return site_out + return site + + def request_data(site, year, month): """Makes a request for each file type until successful or we run out of filetypes. @@ -162,6 +163,11 @@ def fetch(api, site, start, end): return pd.DataFrame() var_columns = [col for col in all_period_data.columns if '_flag' not in col] + power_columns = [col for col in var_columns + if col.startswith('5')] + # adjust power from watts to megawatts + for column in power_columns: + all_period_data[column] = all_period_data[column] / 1000000 all_period_data = all_period_data[var_columns] return all_period_data @@ -216,59 +222,32 @@ def initialize_site_observations(api, site): for variable in srml_variable_map.keys(): matches = [col for col in site_df.columns if col.startswith(variable)] - if variable == '5': - # create an aggregated power observations - if hasattr(site, "modeling_parameters"): - site_power_prefix = _site_power_prefix( - site.modeling_parameters.surface_tilt, - site.modeling_parameters.surface_azimuth, - ) - # limit power to data with matching surface_tilt/azimuth - matches = [match for match in matches - if match.startswith(site_power_prefix)] - if matches: - # from matching raw data files against the - # comprehensive format, it appears that ac power - # measurements are suffixed with a letter, while - # dc is suffixed with a number. - dc_matches = [match for match in matches if isalpha(match[-1])] - ac_matches = [match for match in matches if isnumeric(match[-1])] - observation_extra_parameters = extra_params.copy() - observation_extra_parameters.update({ - 'network_data_label': ','.join(matches)}) - try: - # Here, we pass a name with match instead of - # variable to differentiate between multiple - # observations of the same variable - common.create_observation( - api, site, srml_variable_map[variable], - name=f'{site_name} {srml_variable_map[variable]}', - interval_label='beginning', - extra_params=observation_extra_parameters) - except HTTPError as e: - logger.error( - f'Failed to create {variable} observation at ' - f'Site {site.name}. Error: {e.response.text}') - else: - continue - else: - for match in matches: - observation_extra_parameters = extra_params.copy() - observation_extra_parameters.update({ - 'network_data_label': match}) - try: - # Here, we pass a name with match instead of variable - # to differentiate between multiple observations of - # the same variable - common.create_observation( - api, site, srml_variable_map[variable], - name=f'{site_name} {match}', - interval_label='beginning', - extra_params=observation_extra_parameters) - except HTTPError as e: - logger.error( - f'Failed to create {variable} observation at Site ' - f'{site.name}. Error: {e.response.text}') + for match in matches: + observation_extra_parameters = extra_params.copy() + observation_extra_parameters.update({ + 'network_data_label': match}) + try: + # Here, we pass a name with match instead of variable + # to differentiate between multiple observations of + # the same variable + common.create_observation( + api, site, srml_variable_map[variable], + name=f'{site_name} {match}', + interval_label='beginning', + extra_params=observation_extra_parameters) + except HTTPError as e: + logger.error( + f'Failed to create {variable} observation at Site ' + f'{site.name}. Error: {e.response.text}') + with open(DEFAULT_SITEFILE) as fp: + obs_metadata = json.load(fp)['observations'] + for obs in obs_metadata: + obs_site_extra_params = json.loads(obs['site']['extra_parameters']) + if obs_site_extra_params['network_api_id'] == extra_params[ + 'network_api_id']: + obs['site'] = site + observation = Observation.from_dict(obs) + common.check_and_post_observation(api, observation) def initialize_site_forecasts(api, site): diff --git a/solarforecastarbiter/io/reference_observations/srml_reference_sites.json b/solarforecastarbiter/io/reference_observations/srml_reference_sites.json index 56e3a8d19..e1aa48e40 100644 --- a/solarforecastarbiter/io/reference_observations/srml_reference_sites.json +++ b/solarforecastarbiter/io/reference_observations/srml_reference_sites.json @@ -1,7 +1,7 @@ { "observations": [ { - "extra_parameters": "{\"network_data_label\": \"516A\", \"network\": \"UO SRML\", \"module\": \"Unisolar 68W/ Fronius IG 2000\"}", + "extra_parameters": "{\"network_data_label\": \"516A\", \"module\": \"Unisolar 68W/ Fronius IG 2000\", \"network_api_id\": \"94808.0\", \"attribution\": \"Peterson, J., and Vignola, F., 2017: Structure of a Comprehensive Solar Radiation Dataset. Proceedings of the ASES National Solar Conference 2017. doi: 10.18086/solar.2017.07.02\", \"network\": \"UO SRML\", \"network_api_abbreviation\": \"PS\", \"observation_interval_length\": 1.0}", "interval_label": "beginning", "interval_length": 1.0, "interval_value_type": "interval_mean", @@ -31,7 +31,7 @@ "variable": "dc_power" }, { - "extra_parameters": "{\"network_data_label\": \"5161\", \"network\": \"UO SRML\", \"module\": \"Unisolar 68W/ Fronius IG 2000\"}", + "extra_parameters": "{\"network_data_label\": \"5161\", \"module\": \"Unisolar 68W/ Fronius IG 2000\", \"network_api_id\": \"94808.0\", \"attribution\": \"Peterson, J., and Vignola, F., 2017: Structure of a Comprehensive Solar Radiation Dataset. Proceedings of the ASES National Solar Conference 2017. doi: 10.18086/solar.2017.07.02\", \"network\": \"UO SRML\", \"network_api_abbreviation\": \"PS\", \"observation_interval_length\": 1.0}", "interval_label": "beginning", "interval_length": 1.0, "interval_value_type": "interval_mean", @@ -61,7 +61,7 @@ "variable": "ac_power" }, { - "extra_parameters": "{\"network_data_label\": \"536B\", \"network\": \"UO SRML\", \"module\": \"Evergreen 190W/ Solectra PV1 1800\"}", + "extra_parameters": "{\"network_data_label\": \"536B\", \"module\": \"Evergreen 190W/ Solectra PV1 1800\", \"network_api_id\": \"94808.0\", \"attribution\": \"Peterson, J., and Vignola, F., 2017: Structure of a Comprehensive Solar Radiation Dataset. Proceedings of the ASES National Solar Conference 2017. doi: 10.18086/solar.2017.07.02\", \"network\": \"UO SRML\", \"network_api_abbreviation\": \"PS\", \"observation_interval_length\": 1.0}", "interval_label": "beginning", "interval_length": 1.0, "interval_value_type": "interval_mean", @@ -91,7 +91,7 @@ "variable": "dc_power" }, { - "extra_parameters": "{\"network_data_label\": \"536B\", \"network\": \"UO SRML\", \"module\": \"Evergreen 190W/ Solectra PV1 1800\"}", + "extra_parameters": "{\"network_data_label\": \"536B\", \"module\": \"Evergreen 190W/ Solectra PV1 1800\", \"network_api_id\": \"94808.0\", \"attribution\": \"Peterson, J., and Vignola, F., 2017: Structure of a Comprehensive Solar Radiation Dataset. Proceedings of the ASES National Solar Conference 2017. doi: 10.18086/solar.2017.07.02\", \"network\": \"UO SRML\", \"network_api_abbreviation\": \"PS\", \"observation_interval_length\": 1.0}", "interval_label": "beginning", "interval_length": 1.0, "interval_value_type": "interval_mean", @@ -121,7 +121,7 @@ "variable": "dc_power" }, { - "extra_parameters": "{\"network_data_label\": \"536C\", \"network\": \"UO SRML\", \"module\": \"Photowatt 105W/ PVP 1100\"}", + "extra_parameters": "{\"network_data_label\": \"536C\", \"module\": \"Photowatt 105W/ PVP 1100\", \"network_api_id\": \"94808.0\", \"attribution\": \"Peterson, J., and Vignola, F., 2017: Structure of a Comprehensive Solar Radiation Dataset. Proceedings of the ASES National Solar Conference 2017. doi: 10.18086/solar.2017.07.02\", \"network\": \"UO SRML\", \"network_api_abbreviation\": \"PS\", \"observation_interval_length\": 1.0}", "interval_label": "beginning", "interval_length": 1.0, "interval_value_type": "interval_mean", @@ -151,7 +151,7 @@ "variable": "dc_power" }, { - "extra_parameters": "{\"network_data_label\": \"536C\", \"network\": \"UO SRML\", \"module\": \"Photowatt 105W/ PVP 1100\"}", + "extra_parameters": "{\"network_data_label\": \"536C\", \"module\": \"Photowatt 105W/ PVP 1100\", \"network_api_id\": \"94808.0\", \"attribution\": \"Peterson, J., and Vignola, F., 2017: Structure of a Comprehensive Solar Radiation Dataset. Proceedings of the ASES National Solar Conference 2017. doi: 10.18086/solar.2017.07.02\", \"network\": \"UO SRML\", \"network_api_abbreviation\": \"PS\", \"observation_interval_length\": 1.0}", "interval_label": "beginning", "interval_length": 1.0, "interval_value_type": "interval_mean", @@ -181,7 +181,7 @@ "variable": "dc_power" }, { - "extra_parameters": "{\"network_data_label\": \"536D\", \"network\": \"UO SRML\", \"module\": \"Kaneka 60W/SMA 700\"}", + "extra_parameters": "{\"network_data_label\": \"536D\", \"module\": \"Kaneka 60W/SMA 700\", \"network_api_id\": \"94808.0\", \"attribution\": \"Peterson, J., and Vignola, F., 2017: Structure of a Comprehensive Solar Radiation Dataset. Proceedings of the ASES National Solar Conference 2017. doi: 10.18086/solar.2017.07.02\", \"network\": \"UO SRML\", \"network_api_abbreviation\": \"PS\", \"observation_interval_length\": 1.0}", "interval_label": "beginning", "interval_length": 1.0, "interval_value_type": "interval_mean", @@ -211,7 +211,7 @@ "variable": "dc_power" }, { - "extra_parameters": "{\"network_data_label\": \"536D\", \"network\": \"UO SRML\", \"module\": \"Kaneka 60W/SMA 700\"}", + "extra_parameters": "{\"network_data_label\": \"536D\", \"module\": \"Kaneka 60W/SMA 700\", \"network_api_id\": \"94808.0\", \"attribution\": \"Peterson, J., and Vignola, F., 2017: Structure of a Comprehensive Solar Radiation Dataset. Proceedings of the ASES National Solar Conference 2017. doi: 10.18086/solar.2017.07.02\", \"network\": \"UO SRML\", \"network_api_abbreviation\": \"PS\", \"observation_interval_length\": 1.0}", "interval_label": "beginning", "interval_length": 1.0, "interval_value_type": "interval_mean", @@ -241,11 +241,11 @@ "variable": "dc_power" }, { - "extra_parameters": "{\"network_data_label\": \"5161\", \"network\": \"UO SRML\", \"module\": \"\"}", + "extra_parameters": "{\"network_data_label\": \"5161\", \"module\": \"5kw unspecified\", \"network_api_id\": \"94040.0\", \"attribution\": \"Peterson, J., and Vignola, F., 2017: Structure of a Comprehensive Solar Radiation Dataset. Proceedings of the ASES National Solar Conference 2017. doi: 10.18086/solar.2017.07.02\", \"network\": \"UO SRML\", \"network_api_abbreviation\": \"AS\", \"observation_interval_length\": 1.0}", "interval_label": "beginning", "interval_length": 1.0, "interval_value_type": "interval_mean", - "name": "Ashland OR dc_power ", + "name": "Ashland OR dc_power 5kw", "observation_id": "", "provider": "", "site": { @@ -263,7 +263,7 @@ "temperature_coefficient": 0.3, "tracking_type": "fixed" }, - "name": "Ashland OR", + "name": "Ashland OR PV", "provider": "", "site_id": "" }, @@ -271,11 +271,11 @@ "variable": "dc_power" }, { - "extra_parameters": "{\"network_data_label\": \"5162\", \"network\": \"UO SRML\", \"module\": \"\"}", + "extra_parameters": "{\"network_data_label\": \"5162\", \"module\": \"15kw unspecified\", \"network_api_id\": \"94040.0\", \"attribution\": \"Peterson, J., and Vignola, F., 2017: Structure of a Comprehensive Solar Radiation Dataset. Proceedings of the ASES National Solar Conference 2017. doi: 10.18086/solar.2017.07.02\", \"network\": \"UO SRML\", \"network_api_abbreviation\": \"AS\", \"observation_interval_length\": 1.0}", "interval_label": "beginning", "interval_length": 1.0, "interval_value_type": "interval_mean", - "name": "Ashland OR dc_power ", + "name": "Ashland OR dc_power 15kw", "observation_id": "", "provider": "", "site": { @@ -293,7 +293,7 @@ "temperature_coefficient": 0.3, "tracking_type": "fixed" }, - "name": "Ashland OR", + "name": "Ashland OR PV", "provider": "", "site_id": "" }, @@ -301,7 +301,7 @@ "variable": "dc_power" }, { - "extra_parameters": "{\"network_data_label\": \"5361\", \"network\": \"UO SRML\", \"module\": \"Sharp 175 1050W\"}", + "extra_parameters": "{\"network_data_label\": \"5361\", \"module\": \"Sharp 175 1050W\", \"network_api_id\": \"94259.0\", \"attribution\": \"Peterson, J., and Vignola, F., 2017: Structure of a Comprehensive Solar Radiation Dataset. Proceedings of the ASES National Solar Conference 2017. doi: 10.18086/solar.2017.07.02\", \"network\": \"UO SRML\", \"network_api_abbreviation\": \"KA\", \"observation_interval_length\": 1.0}", "interval_label": "beginning", "interval_length": 1.0, "interval_value_type": "interval_mean", @@ -323,7 +323,7 @@ "temperature_coefficient": 0.3, "tracking_type": "fixed" }, - "name": "Kalapuya High School OR", + "name": "Kalapuya High School OR PV", "provider": "", "site_id": "" }, @@ -331,7 +331,7 @@ "variable": "dc_power" }, { - "extra_parameters": "{\"network_data_label\": \"536A\", \"network\": \"UO SRML\", \"module\": \"Sharp 175 1050W\"}", + "extra_parameters": "{\"network_data_label\": \"536A\", \"module\": \"Sharp 175 1050W\", \"network_api_id\": \"94259.0\", \"attribution\": \"Peterson, J., and Vignola, F., 2017: Structure of a Comprehensive Solar Radiation Dataset. Proceedings of the ASES National Solar Conference 2017. doi: 10.18086/solar.2017.07.02\", \"network\": \"UO SRML\", \"network_api_abbreviation\": \"KA\", \"observation_interval_length\": 1.0}", "interval_label": "beginning", "interval_length": 1.0, "interval_value_type": "interval_mean", @@ -353,7 +353,7 @@ "temperature_coefficient": 0.3, "tracking_type": "fixed" }, - "name": "Kalapuya High School OR", + "name": "Kalapuya High School OR PV", "provider": "", "site_id": "" }, @@ -361,11 +361,11 @@ "variable": "ac_power" }, { - "extra_parameters": "{\"network_data_label\": \"5361\", \"network\": \"UO SRML\", \"module\": \"Sunpower 255, inverter: SPR-3300x 240 VAC Sunpower\"}", + "extra_parameters": "{\"network_data_label\": \"5361\", \"module\": \"Sunpower 255, inverter: SPR-3300x 240 VAC Sunpower\", \"network_api_id\": \"94807.0\", \"attribution\": \"Peterson, J., and Vignola, F., 2017: Structure of a Comprehensive Solar Radiation Dataset. Proceedings of the ASES National Solar Conference 2017. doi: 10.18086/solar.2017.07.02\", \"network\": \"UO SRML\", \"network_api_abbreviation\": \"BD\", \"observation_interval_length\": 1.0}", "interval_label": "beginning", "interval_length": 1.0, "interval_value_type": "interval_mean", - "name": "Bend OR (PV) ac_power Sunpower", + "name": "Bend OR PV ac_power Sunpower", "observation_id": "", "provider": "", "site": { @@ -383,7 +383,7 @@ "temperature_coefficient": 0.3, "tracking_type": "fixed" }, - "name": "Bend OR (PV)", + "name": "Bend OR PV", "provider": "", "site_id": "" }, @@ -391,11 +391,11 @@ "variable": "ac_power" }, { - "extra_parameters": "{\"network_data_label\": \"5369\", \"network\": \"UO SRML\", \"module\": \"Sunpower 255, inverter: SPR-3300x 240 VAC Sunpower\"}", + "extra_parameters": "{\"network_data_label\": \"5369\", \"module\": \"Sunpower 255, inverter: SPR-3300x 240 VAC Sunpower\", \"network_api_id\": \"94807.0\", \"attribution\": \"Peterson, J., and Vignola, F., 2017: Structure of a Comprehensive Solar Radiation Dataset. Proceedings of the ASES National Solar Conference 2017. doi: 10.18086/solar.2017.07.02\", \"network\": \"UO SRML\", \"network_api_abbreviation\": \"BD\", \"observation_interval_length\": 1.0}", "interval_label": "beginning", "interval_length": 1.0, "interval_value_type": "interval_mean", - "name": "Bend OR (PV) dc_power Sunpower", + "name": "Bend OR PV dc_power Sunpower", "observation_id": "", "provider": "", "site": { @@ -413,7 +413,7 @@ "temperature_coefficient": 0.3, "tracking_type": "fixed" }, - "name": "Bend OR (PV)", + "name": "Bend OR PV", "provider": "", "site_id": "" }, @@ -437,7 +437,7 @@ "temperature_coefficient": 0.3, "tracking_type": "fixed" }, - "name": "Ashland OR", + "name": "Ashland OR PV", "provider": "", "site_id": "" }, @@ -456,7 +456,7 @@ "temperature_coefficient": 0.3, "tracking_type": "fixed" }, - "name": "Kalapuya High School OR", + "name": "Kalapuya High School OR PV", "provider": "", "site_id": "" }, @@ -513,9 +513,9 @@ "temperature_coefficient": 0.3, "tracking_type": "fixed" }, - "name": "Bend OR (PV)", + "name": "Bend OR PV", "provider": "", "site_id": "" } ] -} \ No newline at end of file +} From 03d5d3524cc4ed1953e9fdd5a82f3e7c351d9960 Mon Sep 17 00:00:00 2001 From: Leland Boeman Date: Wed, 13 May 2020 13:02:21 -0700 Subject: [PATCH 05/25] add description of reference_observation system --- docs/source/index.rst | 1 + docs/source/reference-observations.rst | 128 ++++++++++++++++++ docs/source/whatsnew/1.0.0rc1.rst | 4 + .../io/reference_observations/srml.py | 1 + 4 files changed, 134 insertions(+) create mode 100644 docs/source/reference-observations.rst diff --git a/docs/source/index.rst b/docs/source/index.rst index c82148356..0c5fb8e4d 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -16,6 +16,7 @@ for more information about the Solar Forecast Arbiter project. installation contributing + reference-observations reference-forecasts api cli diff --git a/docs/source/reference-observations.rst b/docs/source/reference-observations.rst new file mode 100644 index 000000000..51c6eae95 --- /dev/null +++ b/docs/source/reference-observations.rst @@ -0,0 +1,128 @@ +.. curentmodule: solarforecastarbiter.io.reference_observations + +###################### +Reference Observations +###################### + +Overview +======== + +The Solar Forecast Arbiter imports reference observation data from multiple +measurement networks. A list of these networks and their Solar Forecast Arbiter +modules can be found in the `Available Network Handlers`_ section. A map of all +of the sites available in the reference dataset can be found on the +`Solar Forecast Arbiter project website `_. + +The :py:mod:`solarforecastarbiter.io.reference_observations` subpackage +contains code for instantiating reference sites and observations and +updating reference observations. + +Structure +========= + +The :py:mod:`solarforecastarbiter.io.reference_observations` subpackage +contains python modules and metadata files in JSON and CSV format. + +Data Files +---------- +* `sfa_reference_sites.csv` + The master list of reference sites. See the comment at the top of this file + for descriptions of its fields. + +* `_reference_sites.json` + Network specific files containing site and observation metadata in the API's + JSON format. These are used when the master CSV does not contain all of + the columns needed to accurately define a site or observation. + +Modules +------- +* :py:mod:`solarforecastarbiter.io.reference_observations.reference_data` + This module coordinates the initialization and update process. It also + contains the `NETWORKHANDLER_MAP` dictionary, which maps network names to + the correct Network Handler. The functions in the module are utilized by the + CLI `referencedata` command. + +* :py:mod:`solarforecastarbiter.io.reference_observations.common` + A module containing network-agnostic utility functions. For things like + posting data to the Solar Forecast Arbiter API and filtering reference data + by network. + +Network Handlers +**************** +Network Handlers are network specific modules that implement the following +funtions. See :py:mod:`solarforecastarbiter.io.reference_observations.surfrad` +for an example. + +* `initialize_site_observations(api, site)` + Create an observation at the site for each variable available from the + network. + * api: :py:class:`solarforecastarbiter.io.api.APISession` + * site: :py:class:`solarforecastarbiter.datamodel.Site` + + +* `initialize_site_forecasts(api, site)` + Create a forecast fore each observation at the site. + * api: :py:class:`solarforecastarbiter.io.api.APISession` + * site: :py:class:`solarforecastarbiter.datamodel.Site` + + +* `update_observation_data(api, sites, observations, start, end)` + Retrieve data from the network then format and post it to each observation + at the site. + * api: :py:class:`solarforecastarbiter.io.api.APISession` + * sites: :py:class:`solarforecastarbiter.datamodel.Site` + * sites: :py:class:`solarforecastarbiter.datamodel.Site` + * start: datetime + * end: datetime + + +* (optional) `adjust_site_parameters(site)` + In instances where the master site CSV does not contain enough metadata about + the site, (e.g. when a PV plant requires `modeling_parameters`) this function + may be used to update the site metadata before it is posted to the API. + * site: dict + + +Available Network Handlers +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +* NOAA (The National Oceanic and Atmospheric Administration) + * SURFRAD: Surface Radiation Budget Network + https://www.esrl.noaa.gov/gmd/grad/surfrad/ + + :py:mod:`solarforecastarbiter.io.reference_observations.surfrad` + + * SOLRAD: + https://www.esrl.noaa.gov/gmd/grad/solrad/index.html + + :py:mod:`solarforecastarbiter.io.reference_observations.solrad` + + * CRN: U.S. Climate Reference Network + https://www.ncdc.noaa.gov/crn/ + + :py:mod:`solarforecastarbiter.io.reference_observations.crn` + +* NREL MIDC: National Renewable Energy Laboratory Measurement and Instrumentation Data Center + https://midcdmz.nrel.gov/ + + :py:mod:`solarforecastarbiter.io.reference_observations.midc` + +* UO SRML: University of Oregon Solar Radiation Monitoring Laboratory + http://solardat.uoregon.edu/ + + :py:mod:`solarforecastarbiter.io.reference_observations.srml` + +* DOE RTC: DOE Regional Test Centers for Solar Technologies + https://pv-dashboard.sandia.gov/ + + :py:mod:`solarforecastarbiter.io.reference_observations.rtc` + +* DOE ARM: DOE Atmospheric Radiation Measurement + https://www.arm.gov/ + + :py:mod:`solarforecastarbiter.io.reference_observations.arm` + +* NREL PVDAQ: National Renewable Energy Laboratory PV Data Acquisition + https://developer.nrel.gov/docs/solar/pvdaq-v3/ + + :py:mod:`solarforecastarbiter.io.reference_observations.pvdaq` diff --git a/docs/source/whatsnew/1.0.0rc1.rst b/docs/source/whatsnew/1.0.0rc1.rst index 0bdf043f6..ad51c7eac 100644 --- a/docs/source/whatsnew/1.0.0rc1.rst +++ b/docs/source/whatsnew/1.0.0rc1.rst @@ -49,6 +49,8 @@ Enhancements 'DAYTIME INTERPOLATED VALUES'``. (:issue:`124`) * A public Docker image is available, and Github Actions use this image for continuous integration tests (:pull:`446`) +* Adjusted University of Oregon Solar Radiation Measurment Laboratory (UO SRML) + to include available pv power data. (:pull:`442`) Bug fixes ~~~~~~~~~ @@ -56,6 +58,8 @@ Bug fixes (:issue:`428`) (:pull:`430`) * Ensure data is sorted from reference data sources before slicing and posting to the API (:pull:`435`) +* Remove UO SMRL sites from reference dataset that have not reported recent + data. (:issue:`436`) Contributors diff --git a/solarforecastarbiter/io/reference_observations/srml.py b/solarforecastarbiter/io/reference_observations/srml.py index 4ca2dcb9d..81d5a3c80 100644 --- a/solarforecastarbiter/io/reference_observations/srml.py +++ b/solarforecastarbiter/io/reference_observations/srml.py @@ -196,6 +196,7 @@ def initialize_site_observations(api, site): labels to differentiate between measurements recorded by different instruments. """ + # Request ~month old data at initialization to ensure we get a response. start = pd.Timestamp.now() - pd.Timedelta('30 days') end = start try: From 97b55ce5f6c32dbcbaf24897ac5cc733b4247b2f Mon Sep 17 00:00:00 2001 From: Leland Boeman Date: Wed, 13 May 2020 13:44:32 -0700 Subject: [PATCH 06/25] edits, add note about API keys --- docs/source/reference-observations.rst | 44 ++++++++++++++++++-------- 1 file changed, 30 insertions(+), 14 deletions(-) diff --git a/docs/source/reference-observations.rst b/docs/source/reference-observations.rst index 51c6eae95..7ec2fd1ef 100644 --- a/docs/source/reference-observations.rst +++ b/docs/source/reference-observations.rst @@ -8,26 +8,36 @@ Overview ======== The Solar Forecast Arbiter imports reference observation data from multiple -measurement networks. A list of these networks and their Solar Forecast Arbiter -modules can be found in the `Available Network Handlers`_ section. A map of all -of the sites available in the reference dataset can be found on the +measurement networks. All of the logic for creating the appropriate Solar +Forecast Arbiter sites and observations and updating observations with new data +from the network can be found in the +:py:mod:`solarforecastarbiter.io.reference_observations` subpackage. Code for +retrieving data from the network's APIs are spread between the +:py:mod:`solarforecastarbiter.io.fetch` subpackage, and the +`PVLib `_ *iotools* +module. + +A list of these networks and their Solar Forecast Arbiter modules can be found +in the `Available Network Handlers`_ section. A map of all of the sites +available in the reference dataset can be found on the `Solar Forecast Arbiter project website `_. -The :py:mod:`solarforecastarbiter.io.reference_observations` subpackage -contains code for instantiating reference sites and observations and -updating reference observations. Structure ========= The :py:mod:`solarforecastarbiter.io.reference_observations` subpackage -contains python modules and metadata files in JSON and CSV format. +contains python modules and data files in JSON and CSV format. Data Files ---------- * `sfa_reference_sites.csv` The master list of reference sites. See the comment at the top of this file - for descriptions of its fields. + for descriptions of its fields. The file contains extra fields that are not + found in the Solar Forecast Arbiter API schema for Sites. These fields are + for use with the source network's API and are stored in the + `extra_parameters` field when the site is created for use in subsequent + updates. * `_reference_sites.json` Network specific files containing site and observation metadata in the API's @@ -49,9 +59,12 @@ Modules Network Handlers **************** -Network Handlers are network specific modules that implement the following -funtions. See :py:mod:`solarforecastarbiter.io.reference_observations.surfrad` -for an example. +Network Handlers are network specific modules that implement a handful of +functions such that they share a common interface. See +:py:mod:`solarforecastarbiter.io.reference_observations.surfrad` for an +example. + +Required network handler functions: * `initialize_site_observations(api, site)` Create an observation at the site for each variable available from the @@ -112,17 +125,20 @@ Available Network Handlers :py:mod:`solarforecastarbiter.io.reference_observations.srml` -* DOE RTC: DOE Regional Test Centers for Solar Technologies +* DOE RTC: DOE Regional Test Centers for Solar Technologies\* https://pv-dashboard.sandia.gov/ :py:mod:`solarforecastarbiter.io.reference_observations.rtc` -* DOE ARM: DOE Atmospheric Radiation Measurement +* DOE ARM: DOE Atmospheric Radiation Measurement\* https://www.arm.gov/ :py:mod:`solarforecastarbiter.io.reference_observations.arm` -* NREL PVDAQ: National Renewable Energy Laboratory PV Data Acquisition +* NREL PVDAQ: National Renewable Energy Laboratory PV Data Acquisition\* https://developer.nrel.gov/docs/solar/pvdaq-v3/ :py:mod:`solarforecastarbiter.io.reference_observations.pvdaq` + +\* Requesting data from these networks requires a valid api key for their +associated api. From c3f5b1d339c4fa6453b54b359ccb98f4fbab30b8 Mon Sep 17 00:00:00 2001 From: Leland Boeman Date: Wed, 13 May 2020 15:49:39 -0700 Subject: [PATCH 07/25] add tests for new code --- .../reference_observations/tests/test_srml.py | 116 ++++++++++++++++++ 1 file changed, 116 insertions(+) diff --git a/solarforecastarbiter/io/reference_observations/tests/test_srml.py b/solarforecastarbiter/io/reference_observations/tests/test_srml.py index 89bd20a0e..cc1d69b97 100644 --- a/solarforecastarbiter/io/reference_observations/tests/test_srml.py +++ b/solarforecastarbiter/io/reference_observations/tests/test_srml.py @@ -1,8 +1,69 @@ import datetime as dt +import pandas as pd import pytest +from pandas import Timestamp from solarforecastarbiter.io.reference_observations import srml +from solarforecastarbiter.datamodel import Site + + +test_site_dict = { + "elevation": 595.0, + "extra_parameters": "{\"network_api_id\": \"94040.0\", \"attribution\": \"Peterson, J., and Vignola, F., 2017: Structure of a Comprehensive Solar Radiation Dataset. Proceedings of the ASES National Solar Conference 2017. doi: 10.18086/solar.2017.07.02\", \"network\": \"UO SRML\", \"network_api_abbreviation\": \"AS\", \"observation_interval_length\": 1.0}", # noqa + "latitude": 42.19, + "longitude": -122.7, + "modeling_parameters": { + "ac_capacity": 0.02, + "ac_loss_factor": 0.0, + "dc_capacity": 0.02, + "dc_loss_factor": 0.0, + "surface_azimuth": 180.0, + "surface_tilt": 15.0, + "temperature_coefficient": 0.3, + "tracking_type": "fixed" + }, + "name": "Ashland OR PV", + "timezone": "Etc/GMT+8", + "provider": "", + "site_id": "", +} + + +test_site_object = Site.from_dict(test_site_dict) + + +@pytest.fixture() +def test_site(): + return test_site_object + + +srml_df = pd.DataFrame( + {'ghi_1': {Timestamp('2020-03-01 00:00:00-0800', tz='Etc/GMT+8'): 0.0}, + 'ghi_1_flag': {Timestamp('2020-03-01 00:00:00-0800', tz='Etc/GMT+8'): 11}, + 'dni_1': {Timestamp('2020-03-01 00:00:00-0800', tz='Etc/GMT+8'): 0.0}, + 'dni_1_flag': {Timestamp('2020-03-01 00:00:00-0800', tz='Etc/GMT+8'): 11}, + 'dhi_1': {Timestamp('2020-03-01 00:00:00-0800', tz='Etc/GMT+8'): 0.0}, + 'dhi_1_flag': {Timestamp('2020-03-01 00:00:00-0800', tz='Etc/GMT+8'): 11}, + '1161': {Timestamp('2020-03-01 00:00:00-0800', tz='Etc/GMT+8'): 0.0}, + '1161_flag': {Timestamp('2020-03-01 00:00:00-0800', tz='Etc/GMT+8'): 11}, + '5161': {Timestamp('2020-03-01 00:00:00-0800', tz='Etc/GMT+8'): 235.0}, + '5161_flag': {Timestamp('2020-03-01 00:00:00-0800', tz='Etc/GMT+8'): 11}, + '5162': {Timestamp('2020-03-01 00:00:00-0800', tz='Etc/GMT+8'): 250.0}, + '5162_flag': {Timestamp('2020-03-01 00:00:00-0800', tz='Etc/GMT+8'): 11}, + 'wind_speed_3': { + Timestamp('2020-03-01 00:00:00-0800', tz='Etc/GMT+8'): 0.1}, + 'wind_speed_3_flag': { + Timestamp('2020-03-01 00:00:00-0800', tz='Etc/GMT+8'): 11}, + 'temp_air_1': { + Timestamp('2020-03-01 00:00:00-0800', tz='Etc/GMT+8'): 2.1}, + 'temp_air_1_flag': { + Timestamp('2020-03-01 00:00:00-0800', tz='Etc/GMT+8'): 11}}) + + +@pytest.fixture() +def test_data(): + return srml_df @pytest.mark.parametrize('start,end,exp', [ @@ -29,8 +90,63 @@ def test_fetch(mocker, single_site, start, end, exp): assert year_month == exp +@pytest.mark.parametrize('start,end', [ + (dt.datetime(2019, 1, 10), dt.datetime(2019, 3, 11)), +]) +def test_fetch_power_conversion( + mocker, single_site, start, end, test_data): + mocker.patch( + 'solarforecastarbiter.io.reference_observations.srml.request_data', + return_value=test_data) + data = srml.fetch('', single_site, start, end) + assert not [col for col in data.columns if '_flag' in col] + assert data['5161'].iloc[0] == 0.000235 + assert data['5162'].iloc[0] == 0.000250 + + def test_fetch_tz(single_site): start = dt.datetime(2019, 1, 1, tzinfo=dt.timezone.utc) with pytest.raises(TypeError): srml.fetch('', single_site, start, dt.datetime(2019, 2, 1)) + + +@pytest.mark.parametrize('api_id,has_power', [ + (94040.0, True), + (94291.0, False), +]) +def test_adjust_parameters(api_id, has_power): + site = srml.adjust_site_parameters({ + 'extra_parameters': { + 'network_api_id': str(api_id), + } + }) + if has_power: + modeling_params = site['modeling_parameters'] + assert modeling_params['ac_capacity'] == 0.02 + assert modeling_params['dc_capacity'] == 0.02 + assert modeling_params['ac_loss_factor'] == 0.0 + assert modeling_params['dc_loss_factor'] == 0.0 + assert modeling_params['surface_azimuth'] == 180.0 + assert modeling_params['surface_tilt'] == 15.0 + assert modeling_params['temperature_coefficient'] == 0.3 + assert modeling_params['tracking_type'] == 'fixed' + else: + assert 'modeling_parameters' not in site + + +def test_init_site_observations( + mocker, test_data, test_site): + mocker.patch( + 'solarforecastarbiter.io.reference_observations.srml.request_data', + return_value=test_data) + mock_create_obs = mocker.patch( + 'solarforecastarbiter.io.reference_observations.srml.common.' + 'create_observation') + mock_chk_post = mocker.patch( + 'solarforecastarbiter.io.reference_observations.srml.common.' + 'check_and_post_observation') + mock_api = mocker.MagicMock() + srml.initialize_site_observations(mock_api, test_site) + assert mock_create_obs.call_count == 5 + assert mock_chk_post.call_count == 2 From 87782785768133747ab50073805fa3b3d8053acd Mon Sep 17 00:00:00 2001 From: Leland Boeman Date: Wed, 13 May 2020 17:02:30 -0700 Subject: [PATCH 08/25] add missing timezone field in json, differentiate between 15 and 30 degree tilt portland pv site --- .../sfa_reference_sites.csv | 3 +- .../srml_reference_sites.json | 146 ++++++++++-------- 2 files changed, 82 insertions(+), 67 deletions(-) diff --git a/solarforecastarbiter/io/reference_observations/sfa_reference_sites.csv b/solarforecastarbiter/io/reference_observations/sfa_reference_sites.csv index 3895bdfa0..f327fcf07 100644 --- a/solarforecastarbiter/io/reference_observations/sfa_reference_sites.csv +++ b/solarforecastarbiter/io/reference_observations/sfa_reference_sites.csv @@ -30,7 +30,8 @@ interval_length,name,latitude,longitude,elevation,network_api_id,network_api_abb 1,UO Solar Awning Eugene OR,44.05,-123.07,150,94255.0,AW,Etc/GMT+8,"Peterson, J., and Vignola, F., 2017: Structure of a Comprehensive Solar Radiation Dataset. Proceedings of the ASES National Solar Conference 2017. doi: 10.18086/solar.2017.07.02",UO SRML 1,Kalapuya High School OR PV,45.0,-120.0,0,94259.0,KA,Etc/GMT+8,"Peterson, J., and Vignola, F., 2017: Structure of a Comprehensive Solar Radiation Dataset. Proceedings of the ASES National Solar Conference 2017. doi: 10.18086/solar.2017.07.02",UO SRML 1,Salem OR,44.92100000000001,-123.01799999999999,62,94806.0,SA,Etc/GMT+8,"Peterson, J., and Vignola, F., 2017: Structure of a Comprehensive Solar Radiation Dataset. Proceedings of the ASES National Solar Conference 2017. doi: 10.18086/solar.2017.07.02",UO SRML -1,Portland OR PV,45.51,-122.69,70,94808.0,PS,Etc/GMT+8,"Peterson, J., and Vignola, F., 2017: Structure of a Comprehensive Solar Radiation Dataset. Proceedings of the ASES National Solar Conference 2017. doi: 10.18086/solar.2017.07.02",UO SRML +1,Portland OR PV 15 deg tilt,45.51,-122.69,70,94808.0(15),PS,Etc/GMT+8,"Peterson, J., and Vignola, F., 2017: Structure of a Comprehensive Solar Radiation Dataset. Proceedings of the ASES National Solar Conference 2017. doi: 10.18086/solar.2017.07.02",UO SRML +1,Portland OR PV 30 deg tilt,45.51,-122.69,70,94808.0,PS,Etc/GMT+8,"Peterson, J., and Vignola, F., 2017: Structure of a Comprehensive Solar Radiation Dataset. Proceedings of the ASES National Solar Conference 2017. doi: 10.18086/solar.2017.07.02",UO SRML 1,Cheney WA,47.49,-117.589,777,94158.0,CY,Etc/GMT+8,"Peterson, J., and Vignola, F., 2017: Structure of a Comprehensive Solar Radiation Dataset. Proceedings of the ASES National Solar Conference 2017. doi: 10.18086/solar.2017.07.02",UO SRML 1,Eugene OR,44.05,-123.07,150,94255.0,EU,Etc/GMT+8,"Peterson, J., and Vignola, F., 2017: Structure of a Comprehensive Solar Radiation Dataset. Proceedings of the ASES National Solar Conference 2017. doi: 10.18086/solar.2017.07.02",UO SRML 1,Burns OR,43.52,-119.02,1265,94170.0,BU,Etc/GMT+8,"Peterson, J., and Vignola, F., 2017: Structure of a Comprehensive Solar Radiation Dataset. Proceedings of the ASES National Solar Conference 2017. doi: 10.18086/solar.2017.07.02",UO SRML diff --git a/solarforecastarbiter/io/reference_observations/srml_reference_sites.json b/solarforecastarbiter/io/reference_observations/srml_reference_sites.json index e1aa48e40..a3bfa1fdb 100644 --- a/solarforecastarbiter/io/reference_observations/srml_reference_sites.json +++ b/solarforecastarbiter/io/reference_observations/srml_reference_sites.json @@ -1,61 +1,63 @@ { "observations": [ { - "extra_parameters": "{\"network_data_label\": \"516A\", \"module\": \"Unisolar 68W/ Fronius IG 2000\", \"network_api_id\": \"94808.0\", \"attribution\": \"Peterson, J., and Vignola, F., 2017: Structure of a Comprehensive Solar Radiation Dataset. Proceedings of the ASES National Solar Conference 2017. doi: 10.18086/solar.2017.07.02\", \"network\": \"UO SRML\", \"network_api_abbreviation\": \"PS\", \"observation_interval_length\": 1.0}", + "extra_parameters": "{\"network_data_label\": \"516A\", \"module\": \"Unisolar 68W/ Fronius IG 2000\", \"network_api_id\": \"94808.0(15)\", \"attribution\": \"Peterson, J., and Vignola, F., 2017: Structure of a Comprehensive Solar Radiation Dataset. Proceedings of the ASES National Solar Conference 2017. doi: 10.18086/solar.2017.07.02\", \"network\": \"UO SRML\", \"network_api_abbreviation\": \"PS\", \"observation_interval_length\": 1.0}", "interval_label": "beginning", "interval_length": 1.0, "interval_value_type": "interval_mean", - "name": "Portland OR PV dc_power Unisolar", + "name": "Portland OR PV 15 deg tilt dc_power Unisolar", "observation_id": "", "provider": "", "site": { "elevation": 70.0, - "extra_parameters": "{\"network_api_id\": \"94808.0\", \"attribution\": \"Peterson, J., and Vignola, F., 2017: Structure of a Comprehensive Solar Radiation Dataset. Proceedings of the ASES National Solar Conference 2017. doi: 10.18086/solar.2017.07.02\", \"network\": \"UO SRML\", \"network_api_abbreviation\": \"PS\", \"observation_interval_length\": 1.0}", + "extra_parameters": "{\"network_api_id\": \"94808.0(15)\", \"attribution\": \"Peterson, J., and Vignola, F., 2017: Structure of a Comprehensive Solar Radiation Dataset. Proceedings of the ASES National Solar Conference 2017. doi: 10.18086/solar.2017.07.02\", \"network\": \"UO SRML\", \"network_api_abbreviation\": \"PS\", \"observation_interval_length\": 1.0, \"modules\": [\"Unisolar 68W/ Fronius IG 2000\"]}", "latitude": 45.51, "longitude": -122.69, "modeling_parameters": { - "ac_capacity": 0.004755, + "ac_capacity": 0.000816, "ac_loss_factor": 0.0, - "dc_capacity": 0.004755, + "dc_capacity": 0.000816, "dc_loss_factor": 0.0, "surface_azimuth": 180.0, - "surface_tilt": 30.0, - "temperature_coefficient": 0.3, + "surface_tilt": 15.0, + "temperature_coefficient": 0, "tracking_type": "fixed" }, - "name": "Portland OR PV", + "name": "Portland OR PV 15 deg tilt", "provider": "", - "site_id": "" + "site_id": "", + "timezone": "Etc/GMT+8" }, "uncertainty": 0, "variable": "dc_power" }, { - "extra_parameters": "{\"network_data_label\": \"5161\", \"module\": \"Unisolar 68W/ Fronius IG 2000\", \"network_api_id\": \"94808.0\", \"attribution\": \"Peterson, J., and Vignola, F., 2017: Structure of a Comprehensive Solar Radiation Dataset. Proceedings of the ASES National Solar Conference 2017. doi: 10.18086/solar.2017.07.02\", \"network\": \"UO SRML\", \"network_api_abbreviation\": \"PS\", \"observation_interval_length\": 1.0}", + "extra_parameters": "{\"network_data_label\": \"5161\", \"module\": \"Unisolar 68W/ Fronius IG 2000\", \"network_api_id\": \"94808.0(15)\", \"attribution\": \"Peterson, J., and Vignola, F., 2017: Structure of a Comprehensive Solar Radiation Dataset. Proceedings of the ASES National Solar Conference 2017. doi: 10.18086/solar.2017.07.02\", \"network\": \"UO SRML\", \"network_api_abbreviation\": \"PS\", \"observation_interval_length\": 1.0}", "interval_label": "beginning", "interval_length": 1.0, "interval_value_type": "interval_mean", - "name": "Portland OR PV ac_power Unisolar", + "name": "Portland OR PV 15 deg tilt ac_power Unisolar", "observation_id": "", "provider": "", "site": { "elevation": 70.0, - "extra_parameters": "{\"network_api_id\": \"94808.0\", \"attribution\": \"Peterson, J., and Vignola, F., 2017: Structure of a Comprehensive Solar Radiation Dataset. Proceedings of the ASES National Solar Conference 2017. doi: 10.18086/solar.2017.07.02\", \"network\": \"UO SRML\", \"network_api_abbreviation\": \"PS\", \"observation_interval_length\": 1.0}", + "extra_parameters": "{\"network_api_id\": \"94808.0(15)\", \"attribution\": \"Peterson, J., and Vignola, F., 2017: Structure of a Comprehensive Solar Radiation Dataset. Proceedings of the ASES National Solar Conference 2017. doi: 10.18086/solar.2017.07.02\", \"network\": \"UO SRML\", \"network_api_abbreviation\": \"PS\", \"observation_interval_length\": 1.0, \"modules\": [\"Unisolar 68W/ Fronius IG 2000\"]}", "latitude": 45.51, "longitude": -122.69, "modeling_parameters": { - "ac_capacity": 0.004755, + "ac_capacity": 0.000816, "ac_loss_factor": 0.0, - "dc_capacity": 0.004755, + "dc_capacity": 0.000816, "dc_loss_factor": 0.0, "surface_azimuth": 180.0, - "surface_tilt": 30.0, - "temperature_coefficient": 0.3, + "surface_tilt": 15.0, + "temperature_coefficient": 0, "tracking_type": "fixed" }, - "name": "Portland OR PV", + "name": "Portland OR PV 15 deg tilt", "provider": "", - "site_id": "" + "site_id": "", + "timezone": "Etc/GMT+8" }, "uncertainty": 0, "variable": "ac_power" @@ -65,12 +67,12 @@ "interval_label": "beginning", "interval_length": 1.0, "interval_value_type": "interval_mean", - "name": "Portland OR PV dc_power Evergreen", + "name": "Portland OR PV 30 deg tilt dc_power Evergreen", "observation_id": "", "provider": "", "site": { "elevation": 70.0, - "extra_parameters": "{\"network_api_id\": \"94808.0\", \"attribution\": \"Peterson, J., and Vignola, F., 2017: Structure of a Comprehensive Solar Radiation Dataset. Proceedings of the ASES National Solar Conference 2017. doi: 10.18086/solar.2017.07.02\", \"network\": \"UO SRML\", \"network_api_abbreviation\": \"PS\", \"observation_interval_length\": 1.0}", + "extra_parameters": "{\"network_api_id\": \"94808.0\", \"attribution\": \"Peterson, J., and Vignola, F., 2017: Structure of a Comprehensive Solar Radiation Dataset. Proceedings of the ASES National Solar Conference 2017. doi: 10.18086/solar.2017.07.02\", \"network\": \"UO SRML\", \"network_api_abbreviation\": \"PS\", \"observation_interval_length\": 1.0, \"modules\": [\"Evergreen 190W/ Solectra PV1 1800\", \"Photowatt 105W/ PVP 1100\", \"Kaneka 60W/SMA 700\", \"Sanyo 195W/ SMA 3000\"]}", "latitude": 45.51, "longitude": -122.69, "modeling_parameters": { @@ -83,9 +85,10 @@ "temperature_coefficient": 0.3, "tracking_type": "fixed" }, - "name": "Portland OR PV", + "name": "Portland OR PV 30 deg tilt", "provider": "", - "site_id": "" + "site_id": "", + "timezone": "Etc/GMT+8" }, "uncertainty": 0, "variable": "dc_power" @@ -95,12 +98,12 @@ "interval_label": "beginning", "interval_length": 1.0, "interval_value_type": "interval_mean", - "name": "Portland OR PV dc_power Evergreen", + "name": "Portland OR PV 30 deg tilt dc_power Evergreen", "observation_id": "", "provider": "", "site": { "elevation": 70.0, - "extra_parameters": "{\"network_api_id\": \"94808.0\", \"attribution\": \"Peterson, J., and Vignola, F., 2017: Structure of a Comprehensive Solar Radiation Dataset. Proceedings of the ASES National Solar Conference 2017. doi: 10.18086/solar.2017.07.02\", \"network\": \"UO SRML\", \"network_api_abbreviation\": \"PS\", \"observation_interval_length\": 1.0}", + "extra_parameters": "{\"network_api_id\": \"94808.0\", \"attribution\": \"Peterson, J., and Vignola, F., 2017: Structure of a Comprehensive Solar Radiation Dataset. Proceedings of the ASES National Solar Conference 2017. doi: 10.18086/solar.2017.07.02\", \"network\": \"UO SRML\", \"network_api_abbreviation\": \"PS\", \"observation_interval_length\": 1.0, \"modules\": [\"Evergreen 190W/ Solectra PV1 1800\", \"Photowatt 105W/ PVP 1100\", \"Kaneka 60W/SMA 700\", \"Sanyo 195W/ SMA 3000\"]}", "latitude": 45.51, "longitude": -122.69, "modeling_parameters": { @@ -113,9 +116,10 @@ "temperature_coefficient": 0.3, "tracking_type": "fixed" }, - "name": "Portland OR PV", + "name": "Portland OR PV 30 deg tilt", "provider": "", - "site_id": "" + "site_id": "", + "timezone": "Etc/GMT+8" }, "uncertainty": 0, "variable": "dc_power" @@ -125,12 +129,12 @@ "interval_label": "beginning", "interval_length": 1.0, "interval_value_type": "interval_mean", - "name": "Portland OR PV dc_power Photowatt", + "name": "Portland OR PV 30 deg tilt dc_power Photowatt", "observation_id": "", "provider": "", "site": { "elevation": 70.0, - "extra_parameters": "{\"network_api_id\": \"94808.0\", \"attribution\": \"Peterson, J., and Vignola, F., 2017: Structure of a Comprehensive Solar Radiation Dataset. Proceedings of the ASES National Solar Conference 2017. doi: 10.18086/solar.2017.07.02\", \"network\": \"UO SRML\", \"network_api_abbreviation\": \"PS\", \"observation_interval_length\": 1.0}", + "extra_parameters": "{\"network_api_id\": \"94808.0\", \"attribution\": \"Peterson, J., and Vignola, F., 2017: Structure of a Comprehensive Solar Radiation Dataset. Proceedings of the ASES National Solar Conference 2017. doi: 10.18086/solar.2017.07.02\", \"network\": \"UO SRML\", \"network_api_abbreviation\": \"PS\", \"observation_interval_length\": 1.0, \"modules\": [\"Evergreen 190W/ Solectra PV1 1800\", \"Photowatt 105W/ PVP 1100\", \"Kaneka 60W/SMA 700\", \"Sanyo 195W/ SMA 3000\"]}", "latitude": 45.51, "longitude": -122.69, "modeling_parameters": { @@ -143,9 +147,10 @@ "temperature_coefficient": 0.3, "tracking_type": "fixed" }, - "name": "Portland OR PV", + "name": "Portland OR PV 30 deg tilt", "provider": "", - "site_id": "" + "site_id": "", + "timezone": "Etc/GMT+8" }, "uncertainty": 0, "variable": "dc_power" @@ -155,12 +160,12 @@ "interval_label": "beginning", "interval_length": 1.0, "interval_value_type": "interval_mean", - "name": "Portland OR PV dc_power Photowatt", + "name": "Portland OR PV 30 deg tilt dc_power Photowatt", "observation_id": "", "provider": "", "site": { "elevation": 70.0, - "extra_parameters": "{\"network_api_id\": \"94808.0\", \"attribution\": \"Peterson, J., and Vignola, F., 2017: Structure of a Comprehensive Solar Radiation Dataset. Proceedings of the ASES National Solar Conference 2017. doi: 10.18086/solar.2017.07.02\", \"network\": \"UO SRML\", \"network_api_abbreviation\": \"PS\", \"observation_interval_length\": 1.0}", + "extra_parameters": "{\"network_api_id\": \"94808.0\", \"attribution\": \"Peterson, J., and Vignola, F., 2017: Structure of a Comprehensive Solar Radiation Dataset. Proceedings of the ASES National Solar Conference 2017. doi: 10.18086/solar.2017.07.02\", \"network\": \"UO SRML\", \"network_api_abbreviation\": \"PS\", \"observation_interval_length\": 1.0, \"modules\": [\"Evergreen 190W/ Solectra PV1 1800\", \"Photowatt 105W/ PVP 1100\", \"Kaneka 60W/SMA 700\", \"Sanyo 195W/ SMA 3000\"]}", "latitude": 45.51, "longitude": -122.69, "modeling_parameters": { @@ -173,9 +178,10 @@ "temperature_coefficient": 0.3, "tracking_type": "fixed" }, - "name": "Portland OR PV", + "name": "Portland OR PV 30 deg tilt", "provider": "", - "site_id": "" + "site_id": "", + "timezone": "Etc/GMT+8" }, "uncertainty": 0, "variable": "dc_power" @@ -185,12 +191,12 @@ "interval_label": "beginning", "interval_length": 1.0, "interval_value_type": "interval_mean", - "name": "Portland OR PV dc_power Kaneka", + "name": "Portland OR PV 30 deg tilt dc_power Kaneka", "observation_id": "", "provider": "", "site": { "elevation": 70.0, - "extra_parameters": "{\"network_api_id\": \"94808.0\", \"attribution\": \"Peterson, J., and Vignola, F., 2017: Structure of a Comprehensive Solar Radiation Dataset. Proceedings of the ASES National Solar Conference 2017. doi: 10.18086/solar.2017.07.02\", \"network\": \"UO SRML\", \"network_api_abbreviation\": \"PS\", \"observation_interval_length\": 1.0}", + "extra_parameters": "{\"network_api_id\": \"94808.0\", \"attribution\": \"Peterson, J., and Vignola, F., 2017: Structure of a Comprehensive Solar Radiation Dataset. Proceedings of the ASES National Solar Conference 2017. doi: 10.18086/solar.2017.07.02\", \"network\": \"UO SRML\", \"network_api_abbreviation\": \"PS\", \"observation_interval_length\": 1.0, \"modules\": [\"Evergreen 190W/ Solectra PV1 1800\", \"Photowatt 105W/ PVP 1100\", \"Kaneka 60W/SMA 700\", \"Sanyo 195W/ SMA 3000\"]}", "latitude": 45.51, "longitude": -122.69, "modeling_parameters": { @@ -203,9 +209,10 @@ "temperature_coefficient": 0.3, "tracking_type": "fixed" }, - "name": "Portland OR PV", + "name": "Portland OR PV 30 deg tilt", "provider": "", - "site_id": "" + "site_id": "", + "timezone": "Etc/GMT+8" }, "uncertainty": 0, "variable": "dc_power" @@ -215,12 +222,12 @@ "interval_label": "beginning", "interval_length": 1.0, "interval_value_type": "interval_mean", - "name": "Portland OR PV dc_power Kaneka", + "name": "Portland OR PV 30 deg tilt dc_power Kaneka", "observation_id": "", "provider": "", "site": { "elevation": 70.0, - "extra_parameters": "{\"network_api_id\": \"94808.0\", \"attribution\": \"Peterson, J., and Vignola, F., 2017: Structure of a Comprehensive Solar Radiation Dataset. Proceedings of the ASES National Solar Conference 2017. doi: 10.18086/solar.2017.07.02\", \"network\": \"UO SRML\", \"network_api_abbreviation\": \"PS\", \"observation_interval_length\": 1.0}", + "extra_parameters": "{\"network_api_id\": \"94808.0\", \"attribution\": \"Peterson, J., and Vignola, F., 2017: Structure of a Comprehensive Solar Radiation Dataset. Proceedings of the ASES National Solar Conference 2017. doi: 10.18086/solar.2017.07.02\", \"network\": \"UO SRML\", \"network_api_abbreviation\": \"PS\", \"observation_interval_length\": 1.0, \"modules\": [\"Evergreen 190W/ Solectra PV1 1800\", \"Photowatt 105W/ PVP 1100\", \"Kaneka 60W/SMA 700\", \"Sanyo 195W/ SMA 3000\"]}", "latitude": 45.51, "longitude": -122.69, "modeling_parameters": { @@ -233,9 +240,10 @@ "temperature_coefficient": 0.3, "tracking_type": "fixed" }, - "name": "Portland OR PV", + "name": "Portland OR PV 30 deg tilt", "provider": "", - "site_id": "" + "site_id": "", + "timezone": "Etc/GMT+8" }, "uncertainty": 0, "variable": "dc_power" @@ -250,7 +258,7 @@ "provider": "", "site": { "elevation": 595.0, - "extra_parameters": "{\"network_api_id\": \"94040.0\", \"attribution\": \"Peterson, J., and Vignola, F., 2017: Structure of a Comprehensive Solar Radiation Dataset. Proceedings of the ASES National Solar Conference 2017. doi: 10.18086/solar.2017.07.02\", \"network\": \"UO SRML\", \"network_api_abbreviation\": \"AS\", \"observation_interval_length\": 1.0}", + "extra_parameters": "{\"network_api_id\": \"94040.0\", \"attribution\": \"Peterson, J., and Vignola, F., 2017: Structure of a Comprehensive Solar Radiation Dataset. Proceedings of the ASES National Solar Conference 2017. doi: 10.18086/solar.2017.07.02\", \"network\": \"UO SRML\", \"network_api_abbreviation\": \"AS\", \"observation_interval_length\": 1.0, \"modules\": [\"5kw unspecified\", \"15kw unspecified\"]}", "latitude": 42.19, "longitude": -122.7, "modeling_parameters": { @@ -263,9 +271,10 @@ "temperature_coefficient": 0.3, "tracking_type": "fixed" }, - "name": "Ashland OR PV", + "name": "Ashland OR", "provider": "", - "site_id": "" + "site_id": "", + "timezone": "Etc/GMT+8" }, "uncertainty": 0, "variable": "dc_power" @@ -280,7 +289,7 @@ "provider": "", "site": { "elevation": 595.0, - "extra_parameters": "{\"network_api_id\": \"94040.0\", \"attribution\": \"Peterson, J., and Vignola, F., 2017: Structure of a Comprehensive Solar Radiation Dataset. Proceedings of the ASES National Solar Conference 2017. doi: 10.18086/solar.2017.07.02\", \"network\": \"UO SRML\", \"network_api_abbreviation\": \"AS\", \"observation_interval_length\": 1.0}", + "extra_parameters": "{\"network_api_id\": \"94040.0\", \"attribution\": \"Peterson, J., and Vignola, F., 2017: Structure of a Comprehensive Solar Radiation Dataset. Proceedings of the ASES National Solar Conference 2017. doi: 10.18086/solar.2017.07.02\", \"network\": \"UO SRML\", \"network_api_abbreviation\": \"AS\", \"observation_interval_length\": 1.0, \"modules\": [\"5kw unspecified\", \"15kw unspecified\"]}", "latitude": 42.19, "longitude": -122.7, "modeling_parameters": { @@ -293,9 +302,10 @@ "temperature_coefficient": 0.3, "tracking_type": "fixed" }, - "name": "Ashland OR PV", + "name": "Ashland OR", "provider": "", - "site_id": "" + "site_id": "", + "timezone": "Etc/GMT+8" }, "uncertainty": 0, "variable": "dc_power" @@ -310,7 +320,7 @@ "provider": "", "site": { "elevation": 0.0, - "extra_parameters": "{\"network_api_id\": \"94259.0\", \"attribution\": \"Peterson, J., and Vignola, F., 2017: Structure of a Comprehensive Solar Radiation Dataset. Proceedings of the ASES National Solar Conference 2017. doi: 10.18086/solar.2017.07.02\", \"network\": \"UO SRML\", \"network_api_abbreviation\": \"KA\", \"observation_interval_length\": 1.0}", + "extra_parameters": "{\"network_api_id\": \"94259.0\", \"attribution\": \"Peterson, J., and Vignola, F., 2017: Structure of a Comprehensive Solar Radiation Dataset. Proceedings of the ASES National Solar Conference 2017. doi: 10.18086/solar.2017.07.02\", \"network\": \"UO SRML\", \"network_api_abbreviation\": \"KA\", \"observation_interval_length\": 1.0, \"modules\": [\"Sharp 175 1050W\"]}", "latitude": 45.0, "longitude": -120.0, "modeling_parameters": { @@ -323,9 +333,10 @@ "temperature_coefficient": 0.3, "tracking_type": "fixed" }, - "name": "Kalapuya High School OR PV", + "name": "Kalapuya High School OR", "provider": "", - "site_id": "" + "site_id": "", + "timezone": "Etc/GMT+8" }, "uncertainty": 0, "variable": "dc_power" @@ -340,7 +351,7 @@ "provider": "", "site": { "elevation": 0.0, - "extra_parameters": "{\"network_api_id\": \"94259.0\", \"attribution\": \"Peterson, J., and Vignola, F., 2017: Structure of a Comprehensive Solar Radiation Dataset. Proceedings of the ASES National Solar Conference 2017. doi: 10.18086/solar.2017.07.02\", \"network\": \"UO SRML\", \"network_api_abbreviation\": \"KA\", \"observation_interval_length\": 1.0}", + "extra_parameters": "{\"network_api_id\": \"94259.0\", \"attribution\": \"Peterson, J., and Vignola, F., 2017: Structure of a Comprehensive Solar Radiation Dataset. Proceedings of the ASES National Solar Conference 2017. doi: 10.18086/solar.2017.07.02\", \"network\": \"UO SRML\", \"network_api_abbreviation\": \"KA\", \"observation_interval_length\": 1.0, \"modules\": [\"Sharp 175 1050W\"]}", "latitude": 45.0, "longitude": -120.0, "modeling_parameters": { @@ -353,9 +364,10 @@ "temperature_coefficient": 0.3, "tracking_type": "fixed" }, - "name": "Kalapuya High School OR PV", + "name": "Kalapuya High School OR", "provider": "", - "site_id": "" + "site_id": "", + "timezone": "Etc/GMT+8" }, "uncertainty": 0, "variable": "ac_power" @@ -365,12 +377,12 @@ "interval_label": "beginning", "interval_length": 1.0, "interval_value_type": "interval_mean", - "name": "Bend OR PV ac_power Sunpower", + "name": "Bend OR (PV) ac_power Sunpower", "observation_id": "", "provider": "", "site": { "elevation": 1124.0, - "extra_parameters": "{\"network_api_id\": \"94807.0\", \"attribution\": \"Peterson, J., and Vignola, F., 2017: Structure of a Comprehensive Solar Radiation Dataset. Proceedings of the ASES National Solar Conference 2017. doi: 10.18086/solar.2017.07.02\", \"network\": \"UO SRML\", \"network_api_abbreviation\": \"BD\", \"observation_interval_length\": 1.0}", + "extra_parameters": "{\"network_api_id\": \"94807.0\", \"attribution\": \"Peterson, J., and Vignola, F., 2017: Structure of a Comprehensive Solar Radiation Dataset. Proceedings of the ASES National Solar Conference 2017. doi: 10.18086/solar.2017.07.02\", \"network\": \"UO SRML\", \"network_api_abbreviation\": \"BD\", \"observation_interval_length\": 1.0, \"modules\": [\"Sunpower 255, inverter: SPR-3300x 240 VAC Sunpower\"]}", "latitude": 44.06, "longitude": -121.31, "modeling_parameters": { @@ -383,9 +395,10 @@ "temperature_coefficient": 0.3, "tracking_type": "fixed" }, - "name": "Bend OR PV", + "name": "Bend OR (PV)", "provider": "", - "site_id": "" + "site_id": "", + "timezone": "Etc/GMT+8" }, "uncertainty": 0, "variable": "ac_power" @@ -395,12 +408,12 @@ "interval_label": "beginning", "interval_length": 1.0, "interval_value_type": "interval_mean", - "name": "Bend OR PV dc_power Sunpower", + "name": "Bend OR (PV) dc_power Sunpower", "observation_id": "", "provider": "", "site": { "elevation": 1124.0, - "extra_parameters": "{\"network_api_id\": \"94807.0\", \"attribution\": \"Peterson, J., and Vignola, F., 2017: Structure of a Comprehensive Solar Radiation Dataset. Proceedings of the ASES National Solar Conference 2017. doi: 10.18086/solar.2017.07.02\", \"network\": \"UO SRML\", \"network_api_abbreviation\": \"BD\", \"observation_interval_length\": 1.0}", + "extra_parameters": "{\"network_api_id\": \"94807.0\", \"attribution\": \"Peterson, J., and Vignola, F., 2017: Structure of a Comprehensive Solar Radiation Dataset. Proceedings of the ASES National Solar Conference 2017. doi: 10.18086/solar.2017.07.02\", \"network\": \"UO SRML\", \"network_api_abbreviation\": \"BD\", \"observation_interval_length\": 1.0, \"modules\": [\"Sunpower 255, inverter: SPR-3300x 240 VAC Sunpower\"]}", "latitude": 44.06, "longitude": -121.31, "modeling_parameters": { @@ -413,9 +426,10 @@ "temperature_coefficient": 0.3, "tracking_type": "fixed" }, - "name": "Bend OR PV", + "name": "Bend OR (PV)", "provider": "", - "site_id": "" + "site_id": "", + "timezone": "Etc/GMT+8" }, "uncertainty": 0, "variable": "dc_power" @@ -437,7 +451,7 @@ "temperature_coefficient": 0.3, "tracking_type": "fixed" }, - "name": "Ashland OR PV", + "name": "Ashland OR", "provider": "", "site_id": "" }, @@ -456,7 +470,7 @@ "temperature_coefficient": 0.3, "tracking_type": "fixed" }, - "name": "Kalapuya High School OR PV", + "name": "Kalapuya High School OR", "provider": "", "site_id": "" }, @@ -513,9 +527,9 @@ "temperature_coefficient": 0.3, "tracking_type": "fixed" }, - "name": "Bend OR PV", + "name": "Bend OR (PV)", "provider": "", "site_id": "" } ] -} +} \ No newline at end of file From 3d49fce29a6967a48bb0f9a92fe92eb6039faeff Mon Sep 17 00:00:00 2001 From: Leland Boeman Date: Thu, 14 May 2020 10:35:55 -0700 Subject: [PATCH 09/25] more test coverage --- .../io/reference_observations/srml.py | 6 +--- .../reference_observations/tests/test_srml.py | 28 +++++++++++++++++++ 2 files changed, 29 insertions(+), 5 deletions(-) diff --git a/solarforecastarbiter/io/reference_observations/srml.py b/solarforecastarbiter/io/reference_observations/srml.py index 81d5a3c80..1656252a2 100644 --- a/solarforecastarbiter/io/reference_observations/srml.py +++ b/solarforecastarbiter/io/reference_observations/srml.py @@ -77,11 +77,7 @@ def request_data(site, year, month): Parameters ---------- - interval_length: int - The number of minutes between each timestep in the data. Used - to lookup filetypes in FILE_TYPE_MAP. - station: string - The two character station abbreviation found in filenames. + site: :py:class:`solarforecastarbiter.datamodel.Site` year: int The year of the data to request. month: int diff --git a/solarforecastarbiter/io/reference_observations/tests/test_srml.py b/solarforecastarbiter/io/reference_observations/tests/test_srml.py index cc1d69b97..6dd5da80f 100644 --- a/solarforecastarbiter/io/reference_observations/tests/test_srml.py +++ b/solarforecastarbiter/io/reference_observations/tests/test_srml.py @@ -1,8 +1,10 @@ import datetime as dt import pandas as pd import pytest +from urllib import error from pandas import Timestamp +from pandas.testing import assert_frame_equal from solarforecastarbiter.io.reference_observations import srml from solarforecastarbiter.datamodel import Site @@ -150,3 +152,29 @@ def test_init_site_observations( srml.initialize_site_observations(mock_api, test_site) assert mock_create_obs.call_count == 5 assert mock_chk_post.call_count == 2 + + +def request_data_test(mocker, exception, test_site, test_data): + get_func = mocker.patch( + 'solarforecastarbiter.io.reference_observations.srml.iotools.' + 'read_srml_month_from_solardat') + get_func.return_value = test_data + data = srml.request_data(test_site, 1, 1) + assert_frame_equal(data, test_data) + + +@pytest.mark.parametrize('exception', [ + pd.errors.EmptyDataError, + error.URLError, +]) +def request_data_test_warnings(mocker, exception, test_site): + logger = mocker.patch( + 'solarforecastarbiter.io.reference_observations.srml.iotools.' + 'logger.warning') + get_func = mocker.patch( + 'solarforecastarbiter.io.reference_observations.srml.iotools.' + 'read_srml_month_from_solardat') + get_func.side_effect = exception() + data = srml.request_data(test_site, 1, 1) + assert logger.call_count == 5 + assert data is None From 1d41714afd468cc311455d7529a98eca683113c3 Mon Sep 17 00:00:00 2001 From: Leland Boeman Date: Fri, 15 May 2020 09:59:19 -0700 Subject: [PATCH 10/25] Update docs/source/reference-observations.rst Co-authored-by: Will Holmgren --- docs/source/reference-observations.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/reference-observations.rst b/docs/source/reference-observations.rst index 7ec2fd1ef..43485c575 100644 --- a/docs/source/reference-observations.rst +++ b/docs/source/reference-observations.rst @@ -14,7 +14,7 @@ from the network can be found in the :py:mod:`solarforecastarbiter.io.reference_observations` subpackage. Code for retrieving data from the network's APIs are spread between the :py:mod:`solarforecastarbiter.io.fetch` subpackage, and the -`PVLib `_ *iotools* +`pvlib python `_ *iotools* module. A list of these networks and their Solar Forecast Arbiter modules can be found From 23b59aba079261b88a6668f9b07943f597292dac Mon Sep 17 00:00:00 2001 From: Leland Boeman Date: Fri, 15 May 2020 09:59:37 -0700 Subject: [PATCH 11/25] Update docs/source/reference-observations.rst Co-authored-by: Will Holmgren --- docs/source/reference-observations.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/reference-observations.rst b/docs/source/reference-observations.rst index 43485c575..54f6801ab 100644 --- a/docs/source/reference-observations.rst +++ b/docs/source/reference-observations.rst @@ -40,7 +40,7 @@ Data Files updates. * `_reference_sites.json` - Network specific files containing site and observation metadata in the API's + Network-specific files containing site and observation metadata in the API's JSON format. These are used when the master CSV does not contain all of the columns needed to accurately define a site or observation. From e635af59fb3273ebe4078a8440dd0e3cdecde6f4 Mon Sep 17 00:00:00 2001 From: Leland Boeman Date: Fri, 15 May 2020 10:00:14 -0700 Subject: [PATCH 12/25] Update docs/source/reference-observations.rst Co-authored-by: Will Holmgren --- docs/source/reference-observations.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/reference-observations.rst b/docs/source/reference-observations.rst index 54f6801ab..2e00cba6c 100644 --- a/docs/source/reference-observations.rst +++ b/docs/source/reference-observations.rst @@ -60,7 +60,7 @@ Modules Network Handlers **************** Network Handlers are network specific modules that implement a handful of -functions such that they share a common interface. See +functions with a common interface. See :py:mod:`solarforecastarbiter.io.reference_observations.surfrad` for an example. From 709cde3658b578af3bdaa10168cb00a45aa1c933 Mon Sep 17 00:00:00 2001 From: Leland Boeman Date: Fri, 15 May 2020 10:00:33 -0700 Subject: [PATCH 13/25] Update docs/source/whatsnew/1.0.0rc1.rst Co-authored-by: Will Holmgren --- docs/source/whatsnew/1.0.0rc1.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/whatsnew/1.0.0rc1.rst b/docs/source/whatsnew/1.0.0rc1.rst index ad51c7eac..aef5e2e20 100644 --- a/docs/source/whatsnew/1.0.0rc1.rst +++ b/docs/source/whatsnew/1.0.0rc1.rst @@ -50,7 +50,7 @@ Enhancements * A public Docker image is available, and Github Actions use this image for continuous integration tests (:pull:`446`) * Adjusted University of Oregon Solar Radiation Measurment Laboratory (UO SRML) - to include available pv power data. (:pull:`442`) + to include available PV power data. (:pull:`442`) Bug fixes ~~~~~~~~~ From ccd603b44cfd52751c548d0680f9f81990a5f419 Mon Sep 17 00:00:00 2001 From: Leland Boeman Date: Fri, 15 May 2020 10:00:44 -0700 Subject: [PATCH 14/25] Update docs/source/whatsnew/1.0.0rc1.rst Co-authored-by: Will Holmgren --- docs/source/whatsnew/1.0.0rc1.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/whatsnew/1.0.0rc1.rst b/docs/source/whatsnew/1.0.0rc1.rst index aef5e2e20..37e0e7a29 100644 --- a/docs/source/whatsnew/1.0.0rc1.rst +++ b/docs/source/whatsnew/1.0.0rc1.rst @@ -59,7 +59,7 @@ Bug fixes * Ensure data is sorted from reference data sources before slicing and posting to the API (:pull:`435`) * Remove UO SMRL sites from reference dataset that have not reported recent - data. (:issue:`436`) + data. (:issue:`436`) (:pull:`442`) Contributors From c1bbd58ee62349269049095dc3be95da6681fb7a Mon Sep 17 00:00:00 2001 From: Leland Boeman Date: Fri, 15 May 2020 10:28:11 -0700 Subject: [PATCH 15/25] Update docs/source/reference-observations.rst Co-authored-by: Will Holmgren --- docs/source/reference-observations.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/reference-observations.rst b/docs/source/reference-observations.rst index 2e00cba6c..5df274fb9 100644 --- a/docs/source/reference-observations.rst +++ b/docs/source/reference-observations.rst @@ -64,7 +64,7 @@ functions with a common interface. See :py:mod:`solarforecastarbiter.io.reference_observations.surfrad` for an example. -Required network handler functions: +The required network handler functions are: * `initialize_site_observations(api, site)` Create an observation at the site for each variable available from the From f4e82cb8f2ea1886add6cecb009b3568828b20c6 Mon Sep 17 00:00:00 2001 From: Leland Boeman Date: Fri, 15 May 2020 12:24:31 -0700 Subject: [PATCH 16/25] update reference obs docs, fix build warnings, reference classes in srml.py docstrings --- docs/source/reference-observations.rst | 67 ++++++++++--------- .../io/reference_observations/srml.py | 20 +++--- 2 files changed, 47 insertions(+), 40 deletions(-) diff --git a/docs/source/reference-observations.rst b/docs/source/reference-observations.rst index 5df274fb9..fcceccd14 100644 --- a/docs/source/reference-observations.rst +++ b/docs/source/reference-observations.rst @@ -25,7 +25,6 @@ available in the reference dataset can be found on the Structure ========= - The :py:mod:`solarforecastarbiter.io.reference_observations` subpackage contains python modules and data files in JSON and CSV format. @@ -40,22 +39,26 @@ Data Files updates. * `_reference_sites.json` - Network-specific files containing site and observation metadata in the API's - JSON format. These are used when the master CSV does not contain all of - the columns needed to accurately define a site or observation. + Network-specific files containing site and observation metadata in the Solar + Forecast Arbiter API's JSON format. These are used when the master CSV does + not contain all of the columns needed to accurately define a site or + observation. Modules ------- * :py:mod:`solarforecastarbiter.io.reference_observations.reference_data` This module coordinates the initialization and update process. It also contains the `NETWORKHANDLER_MAP` dictionary, which maps network names to - the correct Network Handler. The functions in the module are utilized by the - CLI `referencedata` command. + the correct `Network Handlers`_. The functions in the module are utilized by + the CLI `referencedata` command. * :py:mod:`solarforecastarbiter.io.reference_observations.common` - A module containing network-agnostic utility functions. For things like - posting data to the Solar Forecast Arbiter API and filtering reference data - by network. + The `common` module contains utility functions for use throughout the + `reference_data` subpackage. It has useful functions for converting + external data into Solar Forecast Arbiter Datamodel objects and + network-agnostic utilities for preparing and posting data to the Solar + Forecast Arbiter API. Most `Network Handlers`_ rely heavily on these + functions. Network Handlers **************** @@ -69,51 +72,53 @@ The required network handler functions are: * `initialize_site_observations(api, site)` Create an observation at the site for each variable available from the network. - * api: :py:class:`solarforecastarbiter.io.api.APISession` - * site: :py:class:`solarforecastarbiter.datamodel.Site` + + * api: :py:class:`solarforecastarbiter.io.api.APISession` + * site: :py:class:`solarforecastarbiter.datamodel.Site` * `initialize_site_forecasts(api, site)` - Create a forecast fore each observation at the site. - * api: :py:class:`solarforecastarbiter.io.api.APISession` - * site: :py:class:`solarforecastarbiter.datamodel.Site` + Create a forecast for each observation at the site. + + * api: :py:class:`solarforecastarbiter.io.api.APISession` + * site: :py:class:`solarforecastarbiter.datamodel.Site` * `update_observation_data(api, sites, observations, start, end)` Retrieve data from the network then format and post it to each observation at the site. - * api: :py:class:`solarforecastarbiter.io.api.APISession` - * sites: :py:class:`solarforecastarbiter.datamodel.Site` - * sites: :py:class:`solarforecastarbiter.datamodel.Site` - * start: datetime - * end: datetime + + * api: :py:class:`solarforecastarbiter.io.api.APISession` + * sites: list of :py:class:`solarforecastarbiter.datamodel.Site` + * observations: list of :py:class:`solarforecastarbiter.datamodel.Observation` + * start: datetime + * end: datetime * (optional) `adjust_site_parameters(site)` In instances where the master site CSV does not contain enough metadata about the site, (e.g. when a PV plant requires `modeling_parameters`) this function may be used to update the site metadata before it is posted to the API. - * site: dict + + * site: dict Available Network Handlers ^^^^^^^^^^^^^^^^^^^^^^^^^^ +* SURFRAD: NOAA Surface Radiation Budget Network + https://www.esrl.noaa.gov/gmd/grad/surfrad/ -* NOAA (The National Oceanic and Atmospheric Administration) - * SURFRAD: Surface Radiation Budget Network - https://www.esrl.noaa.gov/gmd/grad/surfrad/ - - :py:mod:`solarforecastarbiter.io.reference_observations.surfrad` + :py:mod:`solarforecastarbiter.io.reference_observations.surfrad` - * SOLRAD: - https://www.esrl.noaa.gov/gmd/grad/solrad/index.html +* SOLRAD: NOAA SOLRAD Network + https://www.esrl.noaa.gov/gmd/grad/solrad/index.html - :py:mod:`solarforecastarbiter.io.reference_observations.solrad` + :py:mod:`solarforecastarbiter.io.reference_observations.solrad` - * CRN: U.S. Climate Reference Network - https://www.ncdc.noaa.gov/crn/ +* CRN: NOAA U.S. Climate Reference Network + https://www.ncdc.noaa.gov/crn/ - :py:mod:`solarforecastarbiter.io.reference_observations.crn` + :py:mod:`solarforecastarbiter.io.reference_observations.crn` * NREL MIDC: National Renewable Energy Laboratory Measurement and Instrumentation Data Center https://midcdmz.nrel.gov/ diff --git a/solarforecastarbiter/io/reference_observations/srml.py b/solarforecastarbiter/io/reference_observations/srml.py index 1656252a2..464a7073a 100644 --- a/solarforecastarbiter/io/reference_observations/srml.py +++ b/solarforecastarbiter/io/reference_observations/srml.py @@ -120,10 +120,10 @@ def fetch(api, site, start, end): Parameters ---------- - api : io.APISession + api : :py:class:`solarforecastarbiter.io.api.APISession` An APISession with a valid JWT for accessing the Reference Data user. - site : datamodel.Site + site : :py:class:`solarforecastarbiter.datamodel.Site` Site object with the appropriate metadata. start : datetime The beginning of the period to request data for. @@ -174,9 +174,9 @@ def initialize_site_observations(api, site): Parameters ---------- - api: io.api.APISession + api: :py:class:`solarforecastarbiter.io.api.APISession` - site : datamodel.Site + site : :py:class:`solarforecastarbiter.datamodel.Site The site object for which to create Observations. Notes @@ -253,9 +253,9 @@ def initialize_site_forecasts(api, site): Parameters ---------- - api : solarforecastarbiter.io.api.APISession + api : :py:class:`solarforecastarbiter.io.api.APISession` An active Reference user session. - site : datamodel.Site + site : :py:class:`solarforecastarbiter.datamodel.Site` The site object for which to create Forecasts. """ common.create_forecasts( @@ -267,15 +267,17 @@ def update_observation_data(api, sites, observations, start, end): """Post new observation data to a list of SRML Observations from start to end. - api : solarforecastarbiter.io.api.APISession + api : :py:class:`solarforecastarbiter.io.api.APISession` An active Reference user session. - sites: list + sites: list of :py:class:`solarforecastarbiter.datamodel.Site` List of all reference sites as Objects + observations: list of :py:class:`solarforecastarbiter.datamodel.Observation` + List of all reference observations as Objects start : datetime The beginning of the period to request data for. end : datetime The end of the period to request data for. - """ + """ # noqa srml_sites = common.filter_by_networks(sites, 'UO SRML') for site in srml_sites: common.update_site_observations(api, fetch, site, observations, From 4e5ce6669d60db2b1dc58980cd798f34518c9676 Mon Sep 17 00:00:00 2001 From: Leland Boeman Date: Wed, 20 May 2020 09:43:07 -0700 Subject: [PATCH 17/25] fix mislabeled and broken tests --- .../reference_observations/tests/test_srml.py | 25 +++++++++---------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/solarforecastarbiter/io/reference_observations/tests/test_srml.py b/solarforecastarbiter/io/reference_observations/tests/test_srml.py index 6dd5da80f..58fa33835 100644 --- a/solarforecastarbiter/io/reference_observations/tests/test_srml.py +++ b/solarforecastarbiter/io/reference_observations/tests/test_srml.py @@ -154,11 +154,11 @@ def test_init_site_observations( assert mock_chk_post.call_count == 2 -def request_data_test(mocker, exception, test_site, test_data): - get_func = mocker.patch( - 'solarforecastarbiter.io.reference_observations.srml.iotools.' - 'read_srml_month_from_solardat') - get_func.return_value = test_data +def test_request_data(mocker, test_site, test_data): + mocked_iotools = mocker.patch( + 'solarforecastarbiter.io.reference_observations.srml.iotools') + mocked_iotools.read_srml_month_from_solardat = mocker.MagicMock( + return_value = test_data) data = srml.request_data(test_site, 1, 1) assert_frame_equal(data, test_data) @@ -167,14 +167,13 @@ def request_data_test(mocker, exception, test_site, test_data): pd.errors.EmptyDataError, error.URLError, ]) -def request_data_test_warnings(mocker, exception, test_site): +def test_request_data_warnings(mocker, exception, test_site): + mocked_iotools = mocker.patch( + 'solarforecastarbiter.io.reference_observations.srml.iotools') + mocked_iotools.read_srml_month_from_solardat = mocker.MagicMock( + side_effect = exception('error')) logger = mocker.patch( - 'solarforecastarbiter.io.reference_observations.srml.iotools.' - 'logger.warning') - get_func = mocker.patch( - 'solarforecastarbiter.io.reference_observations.srml.iotools.' - 'read_srml_month_from_solardat') - get_func.side_effect = exception() + 'solarforecastarbiter.io.reference_observations.srml.logger') data = srml.request_data(test_site, 1, 1) - assert logger.call_count == 5 + assert logger.warning.call_count == 3 assert data is None From 00806ad8c3ca8ff673298d73359a1fd3fb1a675e Mon Sep 17 00:00:00 2001 From: Leland Boeman Date: Wed, 20 May 2020 09:53:50 -0700 Subject: [PATCH 18/25] flake8 --- .../io/reference_observations/tests/test_srml.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/solarforecastarbiter/io/reference_observations/tests/test_srml.py b/solarforecastarbiter/io/reference_observations/tests/test_srml.py index 58fa33835..89dbe3698 100644 --- a/solarforecastarbiter/io/reference_observations/tests/test_srml.py +++ b/solarforecastarbiter/io/reference_observations/tests/test_srml.py @@ -158,7 +158,7 @@ def test_request_data(mocker, test_site, test_data): mocked_iotools = mocker.patch( 'solarforecastarbiter.io.reference_observations.srml.iotools') mocked_iotools.read_srml_month_from_solardat = mocker.MagicMock( - return_value = test_data) + return_value=test_data) data = srml.request_data(test_site, 1, 1) assert_frame_equal(data, test_data) @@ -171,7 +171,7 @@ def test_request_data_warnings(mocker, exception, test_site): mocked_iotools = mocker.patch( 'solarforecastarbiter.io.reference_observations.srml.iotools') mocked_iotools.read_srml_month_from_solardat = mocker.MagicMock( - side_effect = exception('error')) + side_effect=exception('error')) logger = mocker.patch( 'solarforecastarbiter.io.reference_observations.srml.logger') data = srml.request_data(test_site, 1, 1) From 5059a65bc39eb46a9b68b35cb243b694c7f816d0 Mon Sep 17 00:00:00 2001 From: Leland Boeman Date: Wed, 27 May 2020 11:41:06 -0700 Subject: [PATCH 19/25] factor out json modeling params loading to common --- .../io/reference_observations/common.py | 28 +++++++++++++++++++ .../io/reference_observations/pvdaq.py | 12 +------- .../io/reference_observations/srml.py | 12 +------- .../reference_observations/tests/conftest.py | 11 ++++++++ .../tests/test_common.py | 13 +++++++++ 5 files changed, 54 insertions(+), 22 deletions(-) diff --git a/solarforecastarbiter/io/reference_observations/common.py b/solarforecastarbiter/io/reference_observations/common.py index db264b25b..4c6d1d8f7 100644 --- a/solarforecastarbiter/io/reference_observations/common.py +++ b/solarforecastarbiter/io/reference_observations/common.py @@ -629,3 +629,31 @@ def create_forecasts(api, site, variables, templates): persist_created = create_persistence_forecasts( api, site, variables, nwp_templates) return nwp_created + persist_created + + +def apply_json_site_parameters(json_sitefile, site): + """Updates site metadata with modeling parameters found in a json file. + + Parameters + ---------- + json_sitefile: str + Absolute path of a json file with a 'sites' key containing a list of + sites in the Solar Forecast Arbiter JSON format. + site: dict + + Returns + ------- + dict + Copy of inputs plus a new key 'modeling_parameters'. + """ + with open(json_sitefile) as fp: + sites_metadata = json.load(fp)['sites'] + site_api_id = str(site['extra_parameters']['network_api_id']) + for site_metadata in sites_metadata: + site_extra_params = json.loads(site_metadata['extra_parameters']) + if str(site_extra_params['network_api_id']) == site_api_id: + site_out = site.copy() + site_out['modeling_parameters'] = site_metadata[ + 'modeling_parameters'] + return site_out + return site diff --git a/solarforecastarbiter/io/reference_observations/pvdaq.py b/solarforecastarbiter/io/reference_observations/pvdaq.py index 5d9178e43..3205ebff3 100644 --- a/solarforecastarbiter/io/reference_observations/pvdaq.py +++ b/solarforecastarbiter/io/reference_observations/pvdaq.py @@ -41,17 +41,7 @@ def adjust_site_parameters(site): -------- solarforecastarbiter.io.reference_observations.site_df_to_dicts """ - with open(DEFAULT_SITEFILE) as fp: - sites_metadata = json.load(fp)['sites'] - site_api_id = int(site['extra_parameters']['network_api_id']) - for site_metadata in sites_metadata: - site_extra_params = json.loads(site_metadata['extra_parameters']) - if site_extra_params['network_api_id'] == site_api_id: - site_out = site.copy() - site_out['modeling_parameters'] = site_metadata[ - 'modeling_parameters'] - site_out['extra_parameters'].update(site_extra_params) - return site_out + return common.apply_json_site_parameters(DEFAULT_SITEFILE, site) def initialize_site_observations(api, site): diff --git a/solarforecastarbiter/io/reference_observations/srml.py b/solarforecastarbiter/io/reference_observations/srml.py index 464a7073a..a1d05c89a 100644 --- a/solarforecastarbiter/io/reference_observations/srml.py +++ b/solarforecastarbiter/io/reference_observations/srml.py @@ -58,17 +58,7 @@ def adjust_site_parameters(site): dict Copy of inputs plus a new key 'modeling_parameters'. """ - with open(DEFAULT_SITEFILE) as fp: - sites_metadata = json.load(fp)['sites'] - site_api_id = site['extra_parameters']['network_api_id'] - for site_metadata in sites_metadata: - site_extra_params = json.loads(site_metadata['extra_parameters']) - if site_extra_params['network_api_id'] == site_api_id: - site_out = site.copy() - site_out['modeling_parameters'] = site_metadata[ - 'modeling_parameters'] - return site_out - return site + return common.apply_json_site_parameters(DEFAULT_SITEFILE, site) def request_data(site, year, month): diff --git a/solarforecastarbiter/io/reference_observations/tests/conftest.py b/solarforecastarbiter/io/reference_observations/tests/conftest.py index e6cbc1dd3..9cc64d33a 100644 --- a/solarforecastarbiter/io/reference_observations/tests/conftest.py +++ b/solarforecastarbiter/io/reference_observations/tests/conftest.py @@ -1,5 +1,8 @@ import copy import json +from pkg_resources import resource_filename, Requirement + + import numpy as np import pandas as pd import pytest @@ -150,3 +153,11 @@ def mock_fetch(mocker, fake_ghi_data): fetch = mocker.MagicMock() fetch.return_value = fake_ghi_data return fetch + + +@pytest.fixture +def test_json_site_file(): + return resource_filename( + Requirement.parse('solarforecastarbiter'), + 'solarforecastarbiter/io/reference_observations/' + 'tests/data/test_site.json') diff --git a/solarforecastarbiter/io/reference_observations/tests/test_common.py b/solarforecastarbiter/io/reference_observations/tests/test_common.py index 98bf73ad7..0dab23174 100644 --- a/solarforecastarbiter/io/reference_observations/tests/test_common.py +++ b/solarforecastarbiter/io/reference_observations/tests/test_common.py @@ -776,3 +776,16 @@ def test_create_forecasts(template_fx, mocker): assert len(fxs) == 3 assert create_nwp.call_count == 1 assert create_nwp.call_args[0][-1] == templates[:1] + common.create_forecasts(api, site, vars_, templates) + + +@pytest.mark.parametrize('params', [ + {'network_api_id': 2}, + {'network_api_id': '2'}, +]) +def test_apply_json_site_parameters_plant(test_json_site_file, params): + new_site = common.apply_json_site_parameters( + test_json_site_file, + {'extra_parameters': params}, + ) + assert 'modeling_parameters' in new_site From 24ea737192120c6439af6f351f55b7e3d143a17c Mon Sep 17 00:00:00 2001 From: Leland Boeman Date: Wed, 27 May 2020 11:48:00 -0700 Subject: [PATCH 20/25] fix rebase error --- .../io/reference_observations/tests/test_common.py | 1 - 1 file changed, 1 deletion(-) diff --git a/solarforecastarbiter/io/reference_observations/tests/test_common.py b/solarforecastarbiter/io/reference_observations/tests/test_common.py index 0dab23174..89372328d 100644 --- a/solarforecastarbiter/io/reference_observations/tests/test_common.py +++ b/solarforecastarbiter/io/reference_observations/tests/test_common.py @@ -776,7 +776,6 @@ def test_create_forecasts(template_fx, mocker): assert len(fxs) == 3 assert create_nwp.call_count == 1 assert create_nwp.call_args[0][-1] == templates[:1] - common.create_forecasts(api, site, vars_, templates) @pytest.mark.parametrize('params', [ From dd4c6931bb716f5a8b976aec68af79c0033c3e4a Mon Sep 17 00:00:00 2001 From: Leland Boeman Date: Wed, 27 May 2020 12:49:14 -0700 Subject: [PATCH 21/25] use inspect to specify file relative to executing test --- .../io/reference_observations/tests/conftest.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/solarforecastarbiter/io/reference_observations/tests/conftest.py b/solarforecastarbiter/io/reference_observations/tests/conftest.py index 9cc64d33a..639d75770 100644 --- a/solarforecastarbiter/io/reference_observations/tests/conftest.py +++ b/solarforecastarbiter/io/reference_observations/tests/conftest.py @@ -1,6 +1,7 @@ import copy +import inspect import json -from pkg_resources import resource_filename, Requirement +import os import numpy as np @@ -11,6 +12,10 @@ from solarforecastarbiter.datamodel import Site, Observation +TEST_DATA_DIR = os.path.dirname( + os.path.abspath(inspect.getfile(inspect.currentframe()))) + + def site_dicts(): return [copy.deepcopy(site) for site in [ { @@ -157,7 +162,4 @@ def mock_fetch(mocker, fake_ghi_data): @pytest.fixture def test_json_site_file(): - return resource_filename( - Requirement.parse('solarforecastarbiter'), - 'solarforecastarbiter/io/reference_observations/' - 'tests/data/test_site.json') + return os.path.join(TEST_DATA_DIR, 'data', 'test_site.json') From 46041362839edbd652d064154c7a78b5ffc96e65 Mon Sep 17 00:00:00 2001 From: Leland Boeman Date: Wed, 27 May 2020 13:27:40 -0700 Subject: [PATCH 22/25] exclude reference obs test data from gitignore --- .gitignore | 2 ++ .../tests/data/test_site.json | 23 +++++++++++++++++++ .../tests/test_common.py | 11 +++++++++ 3 files changed, 36 insertions(+) create mode 100644 solarforecastarbiter/io/reference_observations/tests/data/test_site.json diff --git a/.gitignore b/.gitignore index 505108ace..bbc2240b2 100644 --- a/.gitignore +++ b/.gitignore @@ -113,6 +113,8 @@ venv.bak/ *.json !solarforecastarbiter/io/reference_observations/*.json +!solarforecastarbiter/io/reference_observations/tests/data/*.json + *.html !solarforecastarbiter/reports/templates/*.html *.md diff --git a/solarforecastarbiter/io/reference_observations/tests/data/test_site.json b/solarforecastarbiter/io/reference_observations/tests/data/test_site.json new file mode 100644 index 000000000..97a77ec85 --- /dev/null +++ b/solarforecastarbiter/io/reference_observations/tests/data/test_site.json @@ -0,0 +1,23 @@ +{ + "sites": [ + { + "elevation": 595.0, + "extra_parameters": "{\"network_api_id\": \"2\", \"attribution\": \"\", \"network\": \"TEST\", \"network_api_abbreviation\": \"SITE2\", \"observation_interval_length\": 1.0}", + "latitude": 42.19, + "longitude": -122.7, + "modeling_parameters": { + "ac_capacity": 0.02, + "ac_loss_factor": 0.0, + "dc_capacity": 0.02, + "dc_loss_factor": 0.0, + "surface_azimuth": 180.0, + "surface_tilt": 15.0, + "temperature_coefficient": 0.3, + "tracking_type": "fixed" + }, + "name": "Test data", + "provider": "", + "site_id": "" + } + ] +} diff --git a/solarforecastarbiter/io/reference_observations/tests/test_common.py b/solarforecastarbiter/io/reference_observations/tests/test_common.py index 89372328d..96493c675 100644 --- a/solarforecastarbiter/io/reference_observations/tests/test_common.py +++ b/solarforecastarbiter/io/reference_observations/tests/test_common.py @@ -788,3 +788,14 @@ def test_apply_json_site_parameters_plant(test_json_site_file, params): {'extra_parameters': params}, ) assert 'modeling_parameters' in new_site + + +@pytest.mark.parametrize('params', [ + {'network_api_id': 'not_plant'}, +]) +def test_apply_json_site_parameters_plant(test_json_site_file, params): + new_site = common.apply_json_site_parameters( + test_json_site_file, + {'extra_parameters': params}, + ) + assert 'modeling_parameters' not in new_site From d024d51394b0a28ead801f4433b529b6db5cabd9 Mon Sep 17 00:00:00 2001 From: Leland Boeman Date: Wed, 27 May 2020 13:36:36 -0700 Subject: [PATCH 23/25] change name of non-plant test --- .../io/reference_observations/tests/test_common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/solarforecastarbiter/io/reference_observations/tests/test_common.py b/solarforecastarbiter/io/reference_observations/tests/test_common.py index 96493c675..888bdc99f 100644 --- a/solarforecastarbiter/io/reference_observations/tests/test_common.py +++ b/solarforecastarbiter/io/reference_observations/tests/test_common.py @@ -793,7 +793,7 @@ def test_apply_json_site_parameters_plant(test_json_site_file, params): @pytest.mark.parametrize('params', [ {'network_api_id': 'not_plant'}, ]) -def test_apply_json_site_parameters_plant(test_json_site_file, params): +def test_apply_json_site_parameters_no_params(test_json_site_file, params): new_site = common.apply_json_site_parameters( test_json_site_file, {'extra_parameters': params}, From 647c88a501ca4484c80ad9b52c469b134cf9fc50 Mon Sep 17 00:00:00 2001 From: Leland Boeman Date: Wed, 27 May 2020 14:41:17 -0700 Subject: [PATCH 24/25] replace extra_parameters update in parameter update function --- solarforecastarbiter/io/reference_observations/common.py | 1 + .../io/reference_observations/tests/test_common.py | 7 ++++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/solarforecastarbiter/io/reference_observations/common.py b/solarforecastarbiter/io/reference_observations/common.py index 4c6d1d8f7..f6f589f98 100644 --- a/solarforecastarbiter/io/reference_observations/common.py +++ b/solarforecastarbiter/io/reference_observations/common.py @@ -655,5 +655,6 @@ def apply_json_site_parameters(json_sitefile, site): site_out = site.copy() site_out['modeling_parameters'] = site_metadata[ 'modeling_parameters'] + site_out['extra_parameters'].update(site_extra_params) return site_out return site diff --git a/solarforecastarbiter/io/reference_observations/tests/test_common.py b/solarforecastarbiter/io/reference_observations/tests/test_common.py index 888bdc99f..b20109111 100644 --- a/solarforecastarbiter/io/reference_observations/tests/test_common.py +++ b/solarforecastarbiter/io/reference_observations/tests/test_common.py @@ -788,7 +788,11 @@ def test_apply_json_site_parameters_plant(test_json_site_file, params): {'extra_parameters': params}, ) assert 'modeling_parameters' in new_site - + extra_params = new_site['extra_parameters'] + assert extra_params['network_api_abbreviation'] == 'SITE2' + assert extra_params['attribution'] == "" + assert extra_params['network'] == 'TEST' + assert extra_params['observation_interval_length'] == 1.0 @pytest.mark.parametrize('params', [ {'network_api_id': 'not_plant'}, @@ -799,3 +803,4 @@ def test_apply_json_site_parameters_no_params(test_json_site_file, params): {'extra_parameters': params}, ) assert 'modeling_parameters' not in new_site + assert list(new_site['extra_parameters'].keys()) == ['network_api_id'] From ab34a1688e12eddcd428598b440feb405cc7f3bd Mon Sep 17 00:00:00 2001 From: Leland Boeman Date: Wed, 27 May 2020 14:42:58 -0700 Subject: [PATCH 25/25] flake8 --- .../io/reference_observations/tests/test_common.py | 1 + 1 file changed, 1 insertion(+) diff --git a/solarforecastarbiter/io/reference_observations/tests/test_common.py b/solarforecastarbiter/io/reference_observations/tests/test_common.py index b20109111..e9b32917f 100644 --- a/solarforecastarbiter/io/reference_observations/tests/test_common.py +++ b/solarforecastarbiter/io/reference_observations/tests/test_common.py @@ -794,6 +794,7 @@ def test_apply_json_site_parameters_plant(test_json_site_file, params): assert extra_params['network'] == 'TEST' assert extra_params['observation_interval_length'] == 1.0 + @pytest.mark.parametrize('params', [ {'network_api_id': 'not_plant'}, ])