From d7deb80cdc5d1b63de5b2865a0c5cf24d4655fc1 Mon Sep 17 00:00:00 2001 From: AdamRJensen Date: Mon, 22 Feb 2021 22:14:15 +0100 Subject: [PATCH 01/57] Add cams.get_cams_radiation function --- docs/sphinx/source/api.rst | 1 + docs/sphinx/source/whatsnew/v0.9.0.rst | 3 + pvlib/iotools/__init__.py | 1 + pvlib/iotools/cams.py | 207 +++++++++++++++++++++++++ 4 files changed, 212 insertions(+) create mode 100644 pvlib/iotools/cams.py diff --git a/docs/sphinx/source/api.rst b/docs/sphinx/source/api.rst index 8805d199a4..31204b6f0d 100644 --- a/docs/sphinx/source/api.rst +++ b/docs/sphinx/source/api.rst @@ -484,6 +484,7 @@ relevant to solar energy modeling. iotools.get_pvgis_tmy iotools.read_pvgis_tmy iotools.read_bsrn + iotools.get_cams_mcclear A :py:class:`~pvlib.location.Location` object may be created from metadata in some files. diff --git a/docs/sphinx/source/whatsnew/v0.9.0.rst b/docs/sphinx/source/whatsnew/v0.9.0.rst index 81e7a0c60b..a4e2688bb0 100644 --- a/docs/sphinx/source/whatsnew/v0.9.0.rst +++ b/docs/sphinx/source/whatsnew/v0.9.0.rst @@ -64,6 +64,9 @@ Enhancements ~~~~~~~~~~~~ * Add :func:`~pvlib.iotools.read_bsrn` for reading BSRN solar radiation data files. (:pull:`1145`, :issue:`1015`) +* Add :func:`~pvlib.iotools.get_cams_radiation` for retrieving CAMS McClear + clear-sky radiation time series. + files. (:pull:`1145`, :issue:`1015`) * In :py:class:`~pvlib.modelchain.ModelChain`, attributes which contain output of models are now collected into ``ModelChain.results``. (:pull:`1076`, :issue:`1067`) diff --git a/pvlib/iotools/__init__.py b/pvlib/iotools/__init__.py index ba5d5e8807..737ee66d4d 100644 --- a/pvlib/iotools/__init__.py +++ b/pvlib/iotools/__init__.py @@ -14,3 +14,4 @@ from pvlib.iotools.psm3 import parse_psm3 # noqa: F401 from pvlib.iotools.pvgis import get_pvgis_tmy, read_pvgis_tmy # noqa: F401 from pvlib.iotools.bsrn import read_bsrn # noqa: F401 +from pvlib.iotools.cams import get_cams_radiation # noqa: F401 diff --git a/pvlib/iotools/cams.py b/pvlib/iotools/cams.py new file mode 100644 index 0000000000..c802420623 --- /dev/null +++ b/pvlib/iotools/cams.py @@ -0,0 +1,207 @@ +"""Functions to access data from Copernicus Atmosphere Monitoring Service + (CAMS) radiation service. +.. codeauthor:: Adam R. Jensen +""" + +import pandas as pd +import requests +import io + + +MCCLEAR_COLUMNS = ['Observation period', 'TOA', 'Clear sky GHI', + 'Clear sky BHI', 'Clear sky DHI', 'Clear sky BNI'] + +MCCLEAR_VERBOSE_COLUMNS = ['sza', 'summer/winter split', 'tco3', 'tcwv', + 'AOD BC', 'AOD DU', 'AOD SS', 'AOD OR', 'AOD SU', + 'AOD NI', 'AOD AM', 'alpha', 'Aerosol type', + 'fiso', 'fvol', 'fgeo', 'albedo'] + +# Dictionary mapping CAMS MCCLEAR variables to pvlib names +MCCLEAR_VARIABLE_MAP = { + 'TOA': 'ghi_extra', + 'Clear sky GHI': 'ghi_clear', + 'Clear sky BHI': 'bhi_clear', + 'Clear sky DHI': 'dhi_clear', + 'Clear sky BNI': 'dni_clear', + 'sza': 'solar_zenith', +} + + +# Dictionary mapping Python time steps to CAMS time step format +TIME_STEPS = {'1min': 'PT01M', '15min': 'PT15M', '1h': 'PT01H', '1d': 'P01D', + '1M': 'P01M'} + +TIME_STEPS_HOURS = {'1min': 1/60, '15min': 15/60, '1h': 1, '1d': 24} + + +def get_cams_mcclear(start_date, end_date, latitude, longitude, email, + altitude=None, time_step='1h', time_ref='UT', + integrated=False, label=None, verbose=False, + map_variables=True, server='www.soda-is.com'): + """ + Retrieve time-series of clear-sky global, beam, and diffuse radiation + anywhere in the world from CAMS McClear [1]_ using the WGET service [2]_. + + + Geographical coverage: wordwide + Time coverage: 2004-01-01 to two days ago + Access: free, but requires registration, see [1]_ + Requests: max. 100 per day + + + Parameters + ---------- + start_date: datetime like + First day of the requested period + end_date: datetime like + Last day of the requested period + latitude: float + in decimal degrees, between -90 and 90, north is positive (ISO 19115) + longitude : float + in decimal degrees, between -180 and 180, east is positive (ISO 19115) + altitude: float, default: None + Altitude in meters. If None, then the altitude is determined from the + NASA SRTM database + email: str + Email address linked to a SoDa account + time_step: str, {'1min', '15min', '1h', '1d', '1M'}, default: '1h' + Time step of the time series, either 1 minute, 15 minute, hourly, + daily, or monthly. + time_reference: str, {'UT', 'TST'}, default: 'UT' + 'UT' (universal time) or 'TST' (True Solar Time) + integrated: boolean, default False + Whether to return integrated irradiation values (Wh/m^2) from CAMS or + average irradiance values (W/m^2) as is more commonly used + label: {‘right’, ‘left’}, default: None + Which bin edge label to label bucket with. The default is ‘left’ for + all frequency offsets except for ‘M’ which has a default of ‘right’. + verbose: boolean, default: False + Verbose mode outputs additional parameters (aerosols). Only avaiable + for 1 minute and universal time. See [1] for parameter description. + map_variables: bool, default: True + When true, renames columns of the Dataframe to pvlib variable names + where applicable. See variable MCCLEAR_VARIABLE_MAP. + server: str, default: 'www.soda-is.com' + Main server (www.soda-is.com) or backup mirror server (pro.soda-is.com) + + + Notes + ---------- + The returned data Dataframe includes the following fields: + + ======================= ====== ========================================== + Key, mapped key Format Description + ======================= ====== ========================================== + **Mapped field names are returned when the map_variables argument is True** + -------------------------------------------------------------------------- + Observation period str Beginning/end of time period + TOA, ghi_extra float Horizontal radiation at top of atmosphere + Clear sky GHI, ghi_clear float Clear sky global radiation on horizontal + Clear sky BHI, bhi_clear float Clear sky beam radiation on horizontal + Clear sky DHI, dhi_clear float Clear sky diffuse radiation on horizontal + Clear sky BNI, dni_clear float Clear sky beam radiation normal to sun + ======================= ====== ========================================== + + For the returned units see the integrated argument. For description of + additional output parameters in verbose mode, see [1]. + + Note that it is recommended to specify the latitude and longitude to at + least the fourth decimal place. + + Variables corresponding to standard pvlib variables are renamed, + e.g. `sza` becomes `solar_zenith`. See the + `pvlib.iotools.cams.MCCLEAR_VARIABLE_MAP` dict for the complete mapping. + + + References + ---------- + .. [1] `CAMS McClear Service Info + `_ + .. [2] `CAMS McClear Automatic Access + `_ + """ + + if time_step in TIME_STEPS.keys(): + time_step_str = TIME_STEPS[time_step] + else: + print('WARNING: time step not recognized, 1 hour time step used!') + time_step_str = 'PT01H' + + names = MCCLEAR_COLUMNS + if verbose: + if (time_step == '1min') & (time_ref == 'UT'): + names += MCCLEAR_VERBOSE_COLUMNS + else: + verbose = False + print("Verbose mode only supports 1 min. UT time series!") + + if altitude is None: # Let SoDa get elevation from the NASA SRTM database + altitude = -999 + + # Start and end date should be in the format: yyyy-mm-dd + start_date = start_date.strftime('%Y-%m-%d') + end_date = end_date.strftime('%Y-%m-%d') + + email = email.replace('@', '%2540') # Format email address + + # Format verbose variable to the required format: {'true', 'false'} + verbose = str(verbose).lower() + + # Manual format the request url, due to uncommon usage of & and ; in url + url = ("http://{}/service/wps?Service=WPS&Request=Execute&" + "Identifier=get_mcclear&version=1.0.0&RawDataOutput=irradiation&" + "DataInputs=latitude={};longitude={};altitude={};" + "date_begin={};date_end={};time_ref={};summarization={};" + "username={};verbose={}" + ).format(server, latitude, longitude, altitude, start_date, + end_date, time_ref, time_step_str, email, verbose) + + res = requests.get(url) + + # Invalid requests returns helpful XML error message + if res.headers['Content-Type'] == 'application/xml': + print('REQUEST ERROR MESSAGE:') + print(res.text.split('ows:ExceptionText')[1][1:-2]) + + # Check if returned file is a csv data file + elif res.headers['Content-Type'] == 'application/csv': + data = pd.read_csv(io.StringIO(res.content.decode('utf-8')), sep=';', + comment='#', header=None, names=names) + + obs_period = data['Observation period'].str.split('/') + + # Set index as the start observation time (left) and localize to UTC + if (label == 'left') | ((label is None) & (time_step != '1M')): + data.index = pd.to_datetime(obs_period.str[0], utc=True) + # Set index as the stop observation time (right) and localize to UTC + elif (label == 'right') | ((label is None) & (time_step == '1M')): + data.index = pd.to_datetime(obs_period.str[1], utc=True) + + data.index.name = None # Set index name to None + + # Change index for '1d' and '1M' to be date and not datetime + if time_step == '1d': + data.index = data.index.date + elif (time_step == '1M') & (label is not None): + data.index = data.index.date + # For monthly data with 'right' label, the index should be the last + # date of the month and not the first date of the following month + elif (time_step == '1M') & (time_step != 'left'): + data.index = data.index.date - pd.Timestamp(days=1) + + if not integrated: # Convert from Wh/m2 to W/m2 + integrated_cols = MCCLEAR_COLUMNS[1:6] + + if time_step == '1M': + time_delta = (pd.to_datetime(obs_period.str[1]) + - pd.to_datetime(obs_period.str[0])) + hours = time_delta.dt.total_seconds()/60/60 + data[integrated_cols] = data[integrated_cols] / hours + else: + data[integrated_cols] = (data[integrated_cols] / + TIME_STEPS_HOURS[time_step]) + + if map_variables: + data = data.rename(columns=MCCLEAR_VARIABLE_MAP) + + return data From 510f08ef8b2d0ee543c197a1433c6294ce410cde Mon Sep 17 00:00:00 2001 From: AdamRJensen Date: Mon, 22 Feb 2021 22:14:29 +0100 Subject: [PATCH 02/57] Revert "Add cams.get_cams_radiation function" This reverts commit d7deb80cdc5d1b63de5b2865a0c5cf24d4655fc1. --- docs/sphinx/source/api.rst | 1 - docs/sphinx/source/whatsnew/v0.9.0.rst | 3 - pvlib/iotools/__init__.py | 1 - pvlib/iotools/cams.py | 207 ------------------------- 4 files changed, 212 deletions(-) delete mode 100644 pvlib/iotools/cams.py diff --git a/docs/sphinx/source/api.rst b/docs/sphinx/source/api.rst index 31204b6f0d..8805d199a4 100644 --- a/docs/sphinx/source/api.rst +++ b/docs/sphinx/source/api.rst @@ -484,7 +484,6 @@ relevant to solar energy modeling. iotools.get_pvgis_tmy iotools.read_pvgis_tmy iotools.read_bsrn - iotools.get_cams_mcclear A :py:class:`~pvlib.location.Location` object may be created from metadata in some files. diff --git a/docs/sphinx/source/whatsnew/v0.9.0.rst b/docs/sphinx/source/whatsnew/v0.9.0.rst index a4e2688bb0..81e7a0c60b 100644 --- a/docs/sphinx/source/whatsnew/v0.9.0.rst +++ b/docs/sphinx/source/whatsnew/v0.9.0.rst @@ -64,9 +64,6 @@ Enhancements ~~~~~~~~~~~~ * Add :func:`~pvlib.iotools.read_bsrn` for reading BSRN solar radiation data files. (:pull:`1145`, :issue:`1015`) -* Add :func:`~pvlib.iotools.get_cams_radiation` for retrieving CAMS McClear - clear-sky radiation time series. - files. (:pull:`1145`, :issue:`1015`) * In :py:class:`~pvlib.modelchain.ModelChain`, attributes which contain output of models are now collected into ``ModelChain.results``. (:pull:`1076`, :issue:`1067`) diff --git a/pvlib/iotools/__init__.py b/pvlib/iotools/__init__.py index 737ee66d4d..ba5d5e8807 100644 --- a/pvlib/iotools/__init__.py +++ b/pvlib/iotools/__init__.py @@ -14,4 +14,3 @@ from pvlib.iotools.psm3 import parse_psm3 # noqa: F401 from pvlib.iotools.pvgis import get_pvgis_tmy, read_pvgis_tmy # noqa: F401 from pvlib.iotools.bsrn import read_bsrn # noqa: F401 -from pvlib.iotools.cams import get_cams_radiation # noqa: F401 diff --git a/pvlib/iotools/cams.py b/pvlib/iotools/cams.py deleted file mode 100644 index c802420623..0000000000 --- a/pvlib/iotools/cams.py +++ /dev/null @@ -1,207 +0,0 @@ -"""Functions to access data from Copernicus Atmosphere Monitoring Service - (CAMS) radiation service. -.. codeauthor:: Adam R. Jensen -""" - -import pandas as pd -import requests -import io - - -MCCLEAR_COLUMNS = ['Observation period', 'TOA', 'Clear sky GHI', - 'Clear sky BHI', 'Clear sky DHI', 'Clear sky BNI'] - -MCCLEAR_VERBOSE_COLUMNS = ['sza', 'summer/winter split', 'tco3', 'tcwv', - 'AOD BC', 'AOD DU', 'AOD SS', 'AOD OR', 'AOD SU', - 'AOD NI', 'AOD AM', 'alpha', 'Aerosol type', - 'fiso', 'fvol', 'fgeo', 'albedo'] - -# Dictionary mapping CAMS MCCLEAR variables to pvlib names -MCCLEAR_VARIABLE_MAP = { - 'TOA': 'ghi_extra', - 'Clear sky GHI': 'ghi_clear', - 'Clear sky BHI': 'bhi_clear', - 'Clear sky DHI': 'dhi_clear', - 'Clear sky BNI': 'dni_clear', - 'sza': 'solar_zenith', -} - - -# Dictionary mapping Python time steps to CAMS time step format -TIME_STEPS = {'1min': 'PT01M', '15min': 'PT15M', '1h': 'PT01H', '1d': 'P01D', - '1M': 'P01M'} - -TIME_STEPS_HOURS = {'1min': 1/60, '15min': 15/60, '1h': 1, '1d': 24} - - -def get_cams_mcclear(start_date, end_date, latitude, longitude, email, - altitude=None, time_step='1h', time_ref='UT', - integrated=False, label=None, verbose=False, - map_variables=True, server='www.soda-is.com'): - """ - Retrieve time-series of clear-sky global, beam, and diffuse radiation - anywhere in the world from CAMS McClear [1]_ using the WGET service [2]_. - - - Geographical coverage: wordwide - Time coverage: 2004-01-01 to two days ago - Access: free, but requires registration, see [1]_ - Requests: max. 100 per day - - - Parameters - ---------- - start_date: datetime like - First day of the requested period - end_date: datetime like - Last day of the requested period - latitude: float - in decimal degrees, between -90 and 90, north is positive (ISO 19115) - longitude : float - in decimal degrees, between -180 and 180, east is positive (ISO 19115) - altitude: float, default: None - Altitude in meters. If None, then the altitude is determined from the - NASA SRTM database - email: str - Email address linked to a SoDa account - time_step: str, {'1min', '15min', '1h', '1d', '1M'}, default: '1h' - Time step of the time series, either 1 minute, 15 minute, hourly, - daily, or monthly. - time_reference: str, {'UT', 'TST'}, default: 'UT' - 'UT' (universal time) or 'TST' (True Solar Time) - integrated: boolean, default False - Whether to return integrated irradiation values (Wh/m^2) from CAMS or - average irradiance values (W/m^2) as is more commonly used - label: {‘right’, ‘left’}, default: None - Which bin edge label to label bucket with. The default is ‘left’ for - all frequency offsets except for ‘M’ which has a default of ‘right’. - verbose: boolean, default: False - Verbose mode outputs additional parameters (aerosols). Only avaiable - for 1 minute and universal time. See [1] for parameter description. - map_variables: bool, default: True - When true, renames columns of the Dataframe to pvlib variable names - where applicable. See variable MCCLEAR_VARIABLE_MAP. - server: str, default: 'www.soda-is.com' - Main server (www.soda-is.com) or backup mirror server (pro.soda-is.com) - - - Notes - ---------- - The returned data Dataframe includes the following fields: - - ======================= ====== ========================================== - Key, mapped key Format Description - ======================= ====== ========================================== - **Mapped field names are returned when the map_variables argument is True** - -------------------------------------------------------------------------- - Observation period str Beginning/end of time period - TOA, ghi_extra float Horizontal radiation at top of atmosphere - Clear sky GHI, ghi_clear float Clear sky global radiation on horizontal - Clear sky BHI, bhi_clear float Clear sky beam radiation on horizontal - Clear sky DHI, dhi_clear float Clear sky diffuse radiation on horizontal - Clear sky BNI, dni_clear float Clear sky beam radiation normal to sun - ======================= ====== ========================================== - - For the returned units see the integrated argument. For description of - additional output parameters in verbose mode, see [1]. - - Note that it is recommended to specify the latitude and longitude to at - least the fourth decimal place. - - Variables corresponding to standard pvlib variables are renamed, - e.g. `sza` becomes `solar_zenith`. See the - `pvlib.iotools.cams.MCCLEAR_VARIABLE_MAP` dict for the complete mapping. - - - References - ---------- - .. [1] `CAMS McClear Service Info - `_ - .. [2] `CAMS McClear Automatic Access - `_ - """ - - if time_step in TIME_STEPS.keys(): - time_step_str = TIME_STEPS[time_step] - else: - print('WARNING: time step not recognized, 1 hour time step used!') - time_step_str = 'PT01H' - - names = MCCLEAR_COLUMNS - if verbose: - if (time_step == '1min') & (time_ref == 'UT'): - names += MCCLEAR_VERBOSE_COLUMNS - else: - verbose = False - print("Verbose mode only supports 1 min. UT time series!") - - if altitude is None: # Let SoDa get elevation from the NASA SRTM database - altitude = -999 - - # Start and end date should be in the format: yyyy-mm-dd - start_date = start_date.strftime('%Y-%m-%d') - end_date = end_date.strftime('%Y-%m-%d') - - email = email.replace('@', '%2540') # Format email address - - # Format verbose variable to the required format: {'true', 'false'} - verbose = str(verbose).lower() - - # Manual format the request url, due to uncommon usage of & and ; in url - url = ("http://{}/service/wps?Service=WPS&Request=Execute&" - "Identifier=get_mcclear&version=1.0.0&RawDataOutput=irradiation&" - "DataInputs=latitude={};longitude={};altitude={};" - "date_begin={};date_end={};time_ref={};summarization={};" - "username={};verbose={}" - ).format(server, latitude, longitude, altitude, start_date, - end_date, time_ref, time_step_str, email, verbose) - - res = requests.get(url) - - # Invalid requests returns helpful XML error message - if res.headers['Content-Type'] == 'application/xml': - print('REQUEST ERROR MESSAGE:') - print(res.text.split('ows:ExceptionText')[1][1:-2]) - - # Check if returned file is a csv data file - elif res.headers['Content-Type'] == 'application/csv': - data = pd.read_csv(io.StringIO(res.content.decode('utf-8')), sep=';', - comment='#', header=None, names=names) - - obs_period = data['Observation period'].str.split('/') - - # Set index as the start observation time (left) and localize to UTC - if (label == 'left') | ((label is None) & (time_step != '1M')): - data.index = pd.to_datetime(obs_period.str[0], utc=True) - # Set index as the stop observation time (right) and localize to UTC - elif (label == 'right') | ((label is None) & (time_step == '1M')): - data.index = pd.to_datetime(obs_period.str[1], utc=True) - - data.index.name = None # Set index name to None - - # Change index for '1d' and '1M' to be date and not datetime - if time_step == '1d': - data.index = data.index.date - elif (time_step == '1M') & (label is not None): - data.index = data.index.date - # For monthly data with 'right' label, the index should be the last - # date of the month and not the first date of the following month - elif (time_step == '1M') & (time_step != 'left'): - data.index = data.index.date - pd.Timestamp(days=1) - - if not integrated: # Convert from Wh/m2 to W/m2 - integrated_cols = MCCLEAR_COLUMNS[1:6] - - if time_step == '1M': - time_delta = (pd.to_datetime(obs_period.str[1]) - - pd.to_datetime(obs_period.str[0])) - hours = time_delta.dt.total_seconds()/60/60 - data[integrated_cols] = data[integrated_cols] / hours - else: - data[integrated_cols] = (data[integrated_cols] / - TIME_STEPS_HOURS[time_step]) - - if map_variables: - data = data.rename(columns=MCCLEAR_VARIABLE_MAP) - - return data From 84e820c8b2a52e278690f5f983ff9114936a4e18 Mon Sep 17 00:00:00 2001 From: AdamRJensen Date: Mon, 22 Feb 2021 22:19:26 +0100 Subject: [PATCH 03/57] Add cams.get_cams_mcclear --- docs/sphinx/source/api.rst | 1 + pvlib/iotools/__init__.py | 1 + pvlib/iotools/cams.py | 207 +++++++++++++++++++++++++++++++++++++ 3 files changed, 209 insertions(+) create mode 100644 pvlib/iotools/cams.py diff --git a/docs/sphinx/source/api.rst b/docs/sphinx/source/api.rst index 8805d199a4..31204b6f0d 100644 --- a/docs/sphinx/source/api.rst +++ b/docs/sphinx/source/api.rst @@ -484,6 +484,7 @@ relevant to solar energy modeling. iotools.get_pvgis_tmy iotools.read_pvgis_tmy iotools.read_bsrn + iotools.get_cams_mcclear A :py:class:`~pvlib.location.Location` object may be created from metadata in some files. diff --git a/pvlib/iotools/__init__.py b/pvlib/iotools/__init__.py index ba5d5e8807..892c18edac 100644 --- a/pvlib/iotools/__init__.py +++ b/pvlib/iotools/__init__.py @@ -14,3 +14,4 @@ from pvlib.iotools.psm3 import parse_psm3 # noqa: F401 from pvlib.iotools.pvgis import get_pvgis_tmy, read_pvgis_tmy # noqa: F401 from pvlib.iotools.bsrn import read_bsrn # noqa: F401 +from pvlib.iotools.cams import get_cams_mcclear # noqa: F401 diff --git a/pvlib/iotools/cams.py b/pvlib/iotools/cams.py new file mode 100644 index 0000000000..c802420623 --- /dev/null +++ b/pvlib/iotools/cams.py @@ -0,0 +1,207 @@ +"""Functions to access data from Copernicus Atmosphere Monitoring Service + (CAMS) radiation service. +.. codeauthor:: Adam R. Jensen +""" + +import pandas as pd +import requests +import io + + +MCCLEAR_COLUMNS = ['Observation period', 'TOA', 'Clear sky GHI', + 'Clear sky BHI', 'Clear sky DHI', 'Clear sky BNI'] + +MCCLEAR_VERBOSE_COLUMNS = ['sza', 'summer/winter split', 'tco3', 'tcwv', + 'AOD BC', 'AOD DU', 'AOD SS', 'AOD OR', 'AOD SU', + 'AOD NI', 'AOD AM', 'alpha', 'Aerosol type', + 'fiso', 'fvol', 'fgeo', 'albedo'] + +# Dictionary mapping CAMS MCCLEAR variables to pvlib names +MCCLEAR_VARIABLE_MAP = { + 'TOA': 'ghi_extra', + 'Clear sky GHI': 'ghi_clear', + 'Clear sky BHI': 'bhi_clear', + 'Clear sky DHI': 'dhi_clear', + 'Clear sky BNI': 'dni_clear', + 'sza': 'solar_zenith', +} + + +# Dictionary mapping Python time steps to CAMS time step format +TIME_STEPS = {'1min': 'PT01M', '15min': 'PT15M', '1h': 'PT01H', '1d': 'P01D', + '1M': 'P01M'} + +TIME_STEPS_HOURS = {'1min': 1/60, '15min': 15/60, '1h': 1, '1d': 24} + + +def get_cams_mcclear(start_date, end_date, latitude, longitude, email, + altitude=None, time_step='1h', time_ref='UT', + integrated=False, label=None, verbose=False, + map_variables=True, server='www.soda-is.com'): + """ + Retrieve time-series of clear-sky global, beam, and diffuse radiation + anywhere in the world from CAMS McClear [1]_ using the WGET service [2]_. + + + Geographical coverage: wordwide + Time coverage: 2004-01-01 to two days ago + Access: free, but requires registration, see [1]_ + Requests: max. 100 per day + + + Parameters + ---------- + start_date: datetime like + First day of the requested period + end_date: datetime like + Last day of the requested period + latitude: float + in decimal degrees, between -90 and 90, north is positive (ISO 19115) + longitude : float + in decimal degrees, between -180 and 180, east is positive (ISO 19115) + altitude: float, default: None + Altitude in meters. If None, then the altitude is determined from the + NASA SRTM database + email: str + Email address linked to a SoDa account + time_step: str, {'1min', '15min', '1h', '1d', '1M'}, default: '1h' + Time step of the time series, either 1 minute, 15 minute, hourly, + daily, or monthly. + time_reference: str, {'UT', 'TST'}, default: 'UT' + 'UT' (universal time) or 'TST' (True Solar Time) + integrated: boolean, default False + Whether to return integrated irradiation values (Wh/m^2) from CAMS or + average irradiance values (W/m^2) as is more commonly used + label: {‘right’, ‘left’}, default: None + Which bin edge label to label bucket with. The default is ‘left’ for + all frequency offsets except for ‘M’ which has a default of ‘right’. + verbose: boolean, default: False + Verbose mode outputs additional parameters (aerosols). Only avaiable + for 1 minute and universal time. See [1] for parameter description. + map_variables: bool, default: True + When true, renames columns of the Dataframe to pvlib variable names + where applicable. See variable MCCLEAR_VARIABLE_MAP. + server: str, default: 'www.soda-is.com' + Main server (www.soda-is.com) or backup mirror server (pro.soda-is.com) + + + Notes + ---------- + The returned data Dataframe includes the following fields: + + ======================= ====== ========================================== + Key, mapped key Format Description + ======================= ====== ========================================== + **Mapped field names are returned when the map_variables argument is True** + -------------------------------------------------------------------------- + Observation period str Beginning/end of time period + TOA, ghi_extra float Horizontal radiation at top of atmosphere + Clear sky GHI, ghi_clear float Clear sky global radiation on horizontal + Clear sky BHI, bhi_clear float Clear sky beam radiation on horizontal + Clear sky DHI, dhi_clear float Clear sky diffuse radiation on horizontal + Clear sky BNI, dni_clear float Clear sky beam radiation normal to sun + ======================= ====== ========================================== + + For the returned units see the integrated argument. For description of + additional output parameters in verbose mode, see [1]. + + Note that it is recommended to specify the latitude and longitude to at + least the fourth decimal place. + + Variables corresponding to standard pvlib variables are renamed, + e.g. `sza` becomes `solar_zenith`. See the + `pvlib.iotools.cams.MCCLEAR_VARIABLE_MAP` dict for the complete mapping. + + + References + ---------- + .. [1] `CAMS McClear Service Info + `_ + .. [2] `CAMS McClear Automatic Access + `_ + """ + + if time_step in TIME_STEPS.keys(): + time_step_str = TIME_STEPS[time_step] + else: + print('WARNING: time step not recognized, 1 hour time step used!') + time_step_str = 'PT01H' + + names = MCCLEAR_COLUMNS + if verbose: + if (time_step == '1min') & (time_ref == 'UT'): + names += MCCLEAR_VERBOSE_COLUMNS + else: + verbose = False + print("Verbose mode only supports 1 min. UT time series!") + + if altitude is None: # Let SoDa get elevation from the NASA SRTM database + altitude = -999 + + # Start and end date should be in the format: yyyy-mm-dd + start_date = start_date.strftime('%Y-%m-%d') + end_date = end_date.strftime('%Y-%m-%d') + + email = email.replace('@', '%2540') # Format email address + + # Format verbose variable to the required format: {'true', 'false'} + verbose = str(verbose).lower() + + # Manual format the request url, due to uncommon usage of & and ; in url + url = ("http://{}/service/wps?Service=WPS&Request=Execute&" + "Identifier=get_mcclear&version=1.0.0&RawDataOutput=irradiation&" + "DataInputs=latitude={};longitude={};altitude={};" + "date_begin={};date_end={};time_ref={};summarization={};" + "username={};verbose={}" + ).format(server, latitude, longitude, altitude, start_date, + end_date, time_ref, time_step_str, email, verbose) + + res = requests.get(url) + + # Invalid requests returns helpful XML error message + if res.headers['Content-Type'] == 'application/xml': + print('REQUEST ERROR MESSAGE:') + print(res.text.split('ows:ExceptionText')[1][1:-2]) + + # Check if returned file is a csv data file + elif res.headers['Content-Type'] == 'application/csv': + data = pd.read_csv(io.StringIO(res.content.decode('utf-8')), sep=';', + comment='#', header=None, names=names) + + obs_period = data['Observation period'].str.split('/') + + # Set index as the start observation time (left) and localize to UTC + if (label == 'left') | ((label is None) & (time_step != '1M')): + data.index = pd.to_datetime(obs_period.str[0], utc=True) + # Set index as the stop observation time (right) and localize to UTC + elif (label == 'right') | ((label is None) & (time_step == '1M')): + data.index = pd.to_datetime(obs_period.str[1], utc=True) + + data.index.name = None # Set index name to None + + # Change index for '1d' and '1M' to be date and not datetime + if time_step == '1d': + data.index = data.index.date + elif (time_step == '1M') & (label is not None): + data.index = data.index.date + # For monthly data with 'right' label, the index should be the last + # date of the month and not the first date of the following month + elif (time_step == '1M') & (time_step != 'left'): + data.index = data.index.date - pd.Timestamp(days=1) + + if not integrated: # Convert from Wh/m2 to W/m2 + integrated_cols = MCCLEAR_COLUMNS[1:6] + + if time_step == '1M': + time_delta = (pd.to_datetime(obs_period.str[1]) + - pd.to_datetime(obs_period.str[0])) + hours = time_delta.dt.total_seconds()/60/60 + data[integrated_cols] = data[integrated_cols] / hours + else: + data[integrated_cols] = (data[integrated_cols] / + TIME_STEPS_HOURS[time_step]) + + if map_variables: + data = data.rename(columns=MCCLEAR_VARIABLE_MAP) + + return data From 0a92f72c0ba455cf9d308d84109c347858d65779 Mon Sep 17 00:00:00 2001 From: AdamRJensen Date: Mon, 22 Feb 2021 22:23:57 +0100 Subject: [PATCH 04/57] Update v0.9.0.rst --- docs/sphinx/source/whatsnew/v0.9.0.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/sphinx/source/whatsnew/v0.9.0.rst b/docs/sphinx/source/whatsnew/v0.9.0.rst index 81e7a0c60b..c254c9e418 100644 --- a/docs/sphinx/source/whatsnew/v0.9.0.rst +++ b/docs/sphinx/source/whatsnew/v0.9.0.rst @@ -64,6 +64,9 @@ Enhancements ~~~~~~~~~~~~ * Add :func:`~pvlib.iotools.read_bsrn` for reading BSRN solar radiation data files. (:pull:`1145`, :issue:`1015`) +* Add :func:`~pvlib.iotools.get_cams_mcclear` for retrieving CAMS McClear + clear-sky radiation time series + files. (:pull:`1172`) * In :py:class:`~pvlib.modelchain.ModelChain`, attributes which contain output of models are now collected into ``ModelChain.results``. (:pull:`1076`, :issue:`1067`) From e8d10989c381d44eced06d3007224073d96ef413 Mon Sep 17 00:00:00 2001 From: AdamRJensen Date: Tue, 23 Feb 2021 23:03:41 +0100 Subject: [PATCH 05/57] Reference correct pull request in whatsnew --- docs/sphinx/source/whatsnew/v0.9.0.rst | 2 +- pvlib/data/cams_mcclear_1min_verbose.csv | 116 +++++++++++++++++++++++ 2 files changed, 117 insertions(+), 1 deletion(-) create mode 100644 pvlib/data/cams_mcclear_1min_verbose.csv diff --git a/docs/sphinx/source/whatsnew/v0.9.0.rst b/docs/sphinx/source/whatsnew/v0.9.0.rst index c254c9e418..497499aa75 100644 --- a/docs/sphinx/source/whatsnew/v0.9.0.rst +++ b/docs/sphinx/source/whatsnew/v0.9.0.rst @@ -66,7 +66,7 @@ Enhancements files. (:pull:`1145`, :issue:`1015`) * Add :func:`~pvlib.iotools.get_cams_mcclear` for retrieving CAMS McClear clear-sky radiation time series - files. (:pull:`1172`) + files. (:pull:`1175`) * In :py:class:`~pvlib.modelchain.ModelChain`, attributes which contain output of models are now collected into ``ModelChain.results``. (:pull:`1076`, :issue:`1067`) diff --git a/pvlib/data/cams_mcclear_1min_verbose.csv b/pvlib/data/cams_mcclear_1min_verbose.csv new file mode 100644 index 0000000000..c57d7cb3ad --- /dev/null +++ b/pvlib/data/cams_mcclear_1min_verbose.csv @@ -0,0 +1,116 @@ +# Coding: utf-8 +# File format version: 4 +# Title: CAMS McClear v3.1 model of clear-sky irradiation. +# Content: A time-series of solar radiation received on horizontal plane and plane always normal to the sun rays at ground level assuming clear sky. +# Calls on the McClear clear-sky model. Returns the global, beam and diffuse irradiations integrated over a selected time step, +# for a selected location (worldwide coverage) and a selected period. +# The research leading to these results has received funding from the European Union within the Copernicus programme. +# Provider: MINES ParisTech (France) +# More information at: http://www.soda-pro.com/web-services/radiation/cams-mcclear +# Date begin (ISO 8601): 2020-06-01T00:00:00.0 +# Date end (ISO 8601): 2020-06-03T00:00:00.0 +# Latitude (positive North, ISO 19115): 55.7906 +# Longitude (positive East, ISO 19115): 12.5251 +# Altitude (m): 39.00 +# Elevation of CAMS cell (m): 28.64 +# Time reference: Universal time (UT) +# +# Encoding partly from D2.8.III.13-14 INSPIRE Data Specification on Atmospheric Conditions and Meteorological Geographical Features - Technical Guidelines (2013-12-10) and CF (Climate and Forecast) metadata (2013-11-11) +# CF Standard Names registry of ObservablePropertyValue +# http://cfconventions.org/Data/cf-standard-names/27/build/cf-standard-name-table.html +# urn:x-inspire:specification:DS-AC-MF:observable-property-name:cf-standard-name:1.6 +# ObservableProperty +# basePhenomenon:"integral_of_surface_downwelling_shortwave_flux_in_air_assuming_clear_sky_wrt_time" +# uom:"Wh m-2" [unit] +# StatisticalMeasure +# statisticalFunction: "sum" +# Summarization (integration) period: 0 year 0 month 0 day 0 h 1 min 0 s +# noValue: nan +# File generated on: 2021-02-23 +# +# Columns: +# 1. Observation period (ISO 8601) +# 2. TOA. Irradiation on horizontal plane at the top of atmosphere (Wh/m2) +# 3. Clear sky GHI. Clear sky global irradiation on horizontal plane at ground level (Wh/m2) +# 4. Clear sky BHI. Clear sky beam irradiation on horizontal plane at ground level (Wh/m2) +# 5. Clear sky DHI. Clear sky diffuse irradiation on horizontal plane at ground level (Wh/m2) +# 6. Clear sky BNI. Clear sky beam irradiation on mobile plane following the sun at normal incidence (Wh/m2) +# 7. sza. Solar zenithal angle for the middle of the summarization (deg) +# 8. summer/winter split. 1.0 means summer, 0.0 means winter +# 9. tco3. Total column content of ozone (Dobson unit) +#10. tcwv. Total column content of water vapour (kg/m2) +#11. AOD BC. Partial aerosol optical depth at 550 nm for black carbon +#12. AOD DU. Partial aerosol optical depth at 550 nm for dust +#13. AOD SS. Partial aerosol optical depth at 550 nm for sea salt +#14. AOD OR. Partial aerosol optical depth at 550 nm for organic matter +#15. AOD SU. Partial aerosol optical depth at 550 nm for sulphate +#16. AOD NI. Partial aerosol optical depth at 550 nm for nitrate +#17. AOD AM. Partial aerosol optical depth at 550 nm for ammonium +#18. alpha. Angstroem coefficient for aerosol +#19. Aerosol type. Obsolete (value is always -1) +#20. fiso. MODIS-like BRDF parameter fiso +#21. fvol. MODIS-like BRDF parameter fvol +#22. fgeo. MODIS-like BRDF parameter fgeo +#23. albedo. Ground albedo +# +# Observation period;TOA;Clear sky GHI;Clear sky BHI;Clear sky DHI;Clear sky BNI;sza;summer/winter split;tco3;tcwv;AOD BC;AOD DU;AOD SS;AOD OR;AOD SU;AOD NI;AOD AM;alpha;Aerosol type;fiso;fvol;fgeo;albedo +2020-06-01T00:00:00.0/2020-06-01T00:01:00.0;0.0000;0.0000;0.0000;0.0000;0.0000;0.0000;0.9709;329.0324;14.1409;0.0053;0.0031;0.0006;0.0127;0.0268;0.0056;0.0015;nan;-1;0.1667;0.0911;0.0267;nan +2020-06-01T00:01:00.0/2020-06-01T00:02:00.0;0.0000;0.0000;0.0000;0.0000;0.0000;0.0000;0.9709;329.0241;14.1498;0.0053;0.0031;0.0006;0.0127;0.0268;0.0056;0.0015;nan;-1;0.1667;0.0911;0.0267;nan +2020-06-01T00:02:00.0/2020-06-01T00:03:00.0;0.0000;0.0000;0.0000;0.0000;0.0000;0.0000;0.9709;329.0158;14.1587;0.0053;0.0031;0.0006;0.0127;0.0268;0.0056;0.0015;nan;-1;0.1667;0.0911;0.0267;nan +2020-06-01T00:03:00.0/2020-06-01T00:04:00.0;0.0000;0.0000;0.0000;0.0000;0.0000;0.0000;0.9709;329.0075;14.1676;0.0053;0.0031;0.0006;0.0127;0.0268;0.0056;0.0015;nan;-1;0.1667;0.0911;0.0267;nan +2020-06-01T00:04:00.0/2020-06-01T00:05:00.0;0.0000;0.0000;0.0000;0.0000;0.0000;0.0000;0.9709;328.9992;14.1765;0.0053;0.0031;0.0006;0.0127;0.0268;0.0055;0.0015;nan;-1;0.1667;0.0911;0.0267;nan +2020-06-01T00:05:00.0/2020-06-01T00:06:00.0;0.0000;0.0000;0.0000;0.0000;0.0000;0.0000;0.9709;328.9909;14.1854;0.0053;0.0031;0.0006;0.0127;0.0268;0.0055;0.0014;nan;-1;0.1667;0.0911;0.0267;nan +2020-06-01T00:06:00.0/2020-06-01T00:07:00.0;0.0000;0.0000;0.0000;0.0000;0.0000;0.0000;0.9709;328.9826;14.1943;0.0053;0.0031;0.0006;0.0127;0.0268;0.0055;0.0014;nan;-1;0.1667;0.0911;0.0267;nan +2020-06-01T00:07:00.0/2020-06-01T00:08:00.0;0.0000;0.0000;0.0000;0.0000;0.0000;0.0000;0.9709;328.9743;14.2032;0.0053;0.0031;0.0006;0.0127;0.0268;0.0055;0.0014;nan;-1;0.1667;0.0911;0.0267;nan +2020-06-01T00:08:00.0/2020-06-01T00:09:00.0;0.0000;0.0000;0.0000;0.0000;0.0000;0.0000;0.9709;328.9660;14.2121;0.0053;0.0031;0.0006;0.0127;0.0268;0.0055;0.0014;nan;-1;0.1667;0.0911;0.0267;nan +2020-06-01T00:09:00.0/2020-06-01T00:10:00.0;0.0000;0.0000;0.0000;0.0000;0.0000;0.0000;0.9709;328.9577;14.2210;0.0053;0.0031;0.0006;0.0128;0.0268;0.0055;0.0014;nan;-1;0.1667;0.0911;0.0267;nan +2020-06-01T00:10:00.0/2020-06-01T00:11:00.0;0.0000;0.0000;0.0000;0.0000;0.0000;0.0000;0.9709;328.9494;14.2299;0.0053;0.0031;0.0006;0.0128;0.0268;0.0055;0.0014;nan;-1;0.1667;0.0911;0.0267;nan +2020-06-01T00:11:00.0/2020-06-01T00:12:00.0;0.0000;0.0000;0.0000;0.0000;0.0000;0.0000;0.9709;328.9411;14.2388;0.0053;0.0031;0.0006;0.0128;0.0268;0.0055;0.0014;nan;-1;0.1667;0.0911;0.0267;nan +2020-06-01T00:12:00.0/2020-06-01T00:13:00.0;0.0000;0.0000;0.0000;0.0000;0.0000;0.0000;0.9709;328.9328;14.2478;0.0053;0.0031;0.0006;0.0128;0.0268;0.0055;0.0014;nan;-1;0.1667;0.0911;0.0267;nan +2020-06-01T00:13:00.0/2020-06-01T00:14:00.0;0.0000;0.0000;0.0000;0.0000;0.0000;0.0000;0.9709;328.9245;14.2567;0.0053;0.0031;0.0006;0.0128;0.0268;0.0055;0.0014;nan;-1;0.1667;0.0911;0.0267;nan +2020-06-01T00:14:00.0/2020-06-01T00:15:00.0;0.0000;0.0000;0.0000;0.0000;0.0000;0.0000;0.9709;328.9162;14.2656;0.0053;0.0031;0.0006;0.0128;0.0268;0.0055;0.0014;nan;-1;0.1667;0.0911;0.0267;nan +2020-06-01T00:15:00.0/2020-06-01T00:16:00.0;0.0000;0.0000;0.0000;0.0000;0.0000;0.0000;0.9709;328.9079;14.2745;0.0053;0.0031;0.0006;0.0128;0.0268;0.0055;0.0014;nan;-1;0.1667;0.0911;0.0267;nan +2020-06-01T00:16:00.0/2020-06-01T00:17:00.0;0.0000;0.0000;0.0000;0.0000;0.0000;0.0000;0.9710;328.8996;14.2834;0.0053;0.0031;0.0006;0.0128;0.0268;0.0055;0.0014;nan;-1;0.1667;0.0911;0.0267;nan +2020-06-01T00:17:00.0/2020-06-01T00:18:00.0;0.0000;0.0000;0.0000;0.0000;0.0000;0.0000;0.9710;328.8913;14.2923;0.0053;0.0031;0.0006;0.0128;0.0268;0.0055;0.0014;nan;-1;0.1667;0.0911;0.0267;nan +2020-06-01T00:18:00.0/2020-06-01T00:19:00.0;0.0000;0.0000;0.0000;0.0000;0.0000;0.0000;0.9710;328.8830;14.3012;0.0053;0.0031;0.0006;0.0128;0.0268;0.0055;0.0014;nan;-1;0.1667;0.0911;0.0267;nan +2020-06-01T00:19:00.0/2020-06-01T00:20:00.0;0.0000;0.0000;0.0000;0.0000;0.0000;0.0000;0.9710;328.8747;14.3101;0.0053;0.0031;0.0006;0.0128;0.0268;0.0055;0.0014;nan;-1;0.1667;0.0911;0.0267;nan +2020-06-01T00:20:00.0/2020-06-01T00:21:00.0;0.0000;0.0000;0.0000;0.0000;0.0000;0.0000;0.9710;328.8664;14.3190;0.0053;0.0031;0.0006;0.0128;0.0268;0.0055;0.0014;nan;-1;0.1667;0.0911;0.0267;nan +2020-06-01T00:21:00.0/2020-06-01T00:22:00.0;0.0000;0.0000;0.0000;0.0000;0.0000;0.0000;0.9710;328.8581;14.3279;0.0054;0.0032;0.0006;0.0128;0.0268;0.0055;0.0014;nan;-1;0.1667;0.0911;0.0267;nan +2020-06-01T00:22:00.0/2020-06-01T00:23:00.0;0.0000;0.0000;0.0000;0.0000;0.0000;0.0000;0.9710;328.8498;14.3368;0.0054;0.0032;0.0006;0.0128;0.0268;0.0054;0.0014;nan;-1;0.1667;0.0911;0.0267;nan +2020-06-01T00:23:00.0/2020-06-01T00:24:00.0;0.0000;0.0000;0.0000;0.0000;0.0000;0.0000;0.9710;328.8415;14.3457;0.0054;0.0032;0.0006;0.0129;0.0268;0.0054;0.0014;nan;-1;0.1667;0.0911;0.0267;nan +2020-06-01T00:24:00.0/2020-06-01T00:25:00.0;0.0000;0.0000;0.0000;0.0000;0.0000;0.0000;0.9710;328.8332;14.3547;0.0054;0.0032;0.0006;0.0129;0.0268;0.0054;0.0014;nan;-1;0.1667;0.0911;0.0267;nan +2020-06-01T00:25:00.0/2020-06-01T00:26:00.0;0.0000;0.0000;0.0000;0.0000;0.0000;0.0000;0.9710;328.8249;14.3636;0.0054;0.0032;0.0006;0.0129;0.0268;0.0054;0.0014;nan;-1;0.1667;0.0911;0.0267;nan +2020-06-01T00:26:00.0/2020-06-01T00:27:00.0;0.0000;0.0000;0.0000;0.0000;0.0000;0.0000;0.9710;328.8166;14.3725;0.0054;0.0032;0.0006;0.0129;0.0268;0.0054;0.0014;nan;-1;0.1667;0.0911;0.0267;nan +2020-06-01T00:27:00.0/2020-06-01T00:28:00.0;0.0000;0.0000;0.0000;0.0000;0.0000;0.0000;0.9710;328.8083;14.3814;0.0054;0.0032;0.0006;0.0129;0.0268;0.0054;0.0014;nan;-1;0.1667;0.0911;0.0267;nan +2020-06-01T00:28:00.0/2020-06-01T00:29:00.0;0.0000;0.0000;0.0000;0.0000;0.0000;0.0000;0.9710;328.8000;14.3903;0.0054;0.0032;0.0006;0.0129;0.0268;0.0054;0.0014;nan;-1;0.1667;0.0911;0.0267;nan +2020-06-01T00:29:00.0/2020-06-01T00:30:00.0;0.0000;0.0000;0.0000;0.0000;0.0000;0.0000;0.9710;328.7917;14.3992;0.0054;0.0032;0.0006;0.0129;0.0268;0.0054;0.0014;nan;-1;0.1667;0.0911;0.0267;nan +2020-06-01T00:30:00.0/2020-06-01T00:31:00.0;0.0000;0.0000;0.0000;0.0000;0.0000;0.0000;0.9710;328.7834;14.4081;0.0054;0.0032;0.0006;0.0129;0.0268;0.0054;0.0014;nan;-1;0.1667;0.0911;0.0267;nan +2020-06-01T00:31:00.0/2020-06-01T00:32:00.0;0.0000;0.0000;0.0000;0.0000;0.0000;0.0000;0.9710;328.7751;14.4170;0.0054;0.0032;0.0006;0.0129;0.0268;0.0054;0.0014;nan;-1;0.1667;0.0911;0.0267;nan +2020-06-01T00:32:00.0/2020-06-01T00:33:00.0;0.0000;0.0000;0.0000;0.0000;0.0000;0.0000;0.9710;328.7668;14.4259;0.0054;0.0032;0.0006;0.0129;0.0268;0.0054;0.0014;nan;-1;0.1667;0.0911;0.0267;nan +2020-06-01T00:33:00.0/2020-06-01T00:34:00.0;0.0000;0.0000;0.0000;0.0000;0.0000;0.0000;0.9710;328.7585;14.4348;0.0054;0.0032;0.0006;0.0129;0.0268;0.0054;0.0014;nan;-1;0.1667;0.0911;0.0267;nan +2020-06-01T00:34:00.0/2020-06-01T00:35:00.0;0.0000;0.0000;0.0000;0.0000;0.0000;0.0000;0.9710;328.7502;14.4437;0.0054;0.0032;0.0006;0.0129;0.0268;0.0054;0.0014;nan;-1;0.1667;0.0911;0.0267;nan +2020-06-01T00:35:00.0/2020-06-01T00:36:00.0;0.0000;0.0000;0.0000;0.0000;0.0000;0.0000;0.9710;328.7419;14.4526;0.0054;0.0032;0.0006;0.0129;0.0268;0.0054;0.0014;nan;-1;0.1667;0.0911;0.0267;nan +2020-06-01T00:36:00.0/2020-06-01T00:37:00.0;0.0000;0.0000;0.0000;0.0000;0.0000;0.0000;0.9710;328.7336;14.4615;0.0054;0.0032;0.0006;0.0130;0.0268;0.0054;0.0014;nan;-1;0.1667;0.0911;0.0267;nan +2020-06-01T00:37:00.0/2020-06-01T00:38:00.0;0.0000;0.0000;0.0000;0.0000;0.0000;0.0000;0.9710;328.7253;14.4705;0.0054;0.0032;0.0006;0.0130;0.0268;0.0054;0.0014;nan;-1;0.1667;0.0911;0.0267;nan +2020-06-01T00:38:00.0/2020-06-01T00:39:00.0;0.0000;0.0000;0.0000;0.0000;0.0000;0.0000;0.9710;328.7170;14.4794;0.0054;0.0032;0.0006;0.0130;0.0267;0.0054;0.0014;nan;-1;0.1667;0.0911;0.0267;nan +2020-06-01T00:39:00.0/2020-06-01T00:40:00.0;0.0000;0.0000;0.0000;0.0000;0.0000;0.0000;0.9710;328.7087;14.4883;0.0054;0.0032;0.0006;0.0130;0.0267;0.0053;0.0014;nan;-1;0.1667;0.0911;0.0267;nan +2020-06-01T00:40:00.0/2020-06-01T00:41:00.0;0.0000;0.0000;0.0000;0.0000;0.0000;0.0000;0.9710;328.7004;14.4972;0.0054;0.0032;0.0006;0.0130;0.0267;0.0053;0.0014;nan;-1;0.1667;0.0911;0.0267;nan +2020-06-01T00:41:00.0/2020-06-01T00:42:00.0;0.0000;0.0000;0.0000;0.0000;0.0000;0.0000;0.9710;328.6921;14.5061;0.0054;0.0032;0.0006;0.0130;0.0267;0.0053;0.0014;nan;-1;0.1667;0.0911;0.0267;nan +2020-06-01T00:42:00.0/2020-06-01T00:43:00.0;0.0000;0.0000;0.0000;0.0000;0.0000;0.0000;0.9710;328.6838;14.5150;0.0054;0.0032;0.0006;0.0130;0.0267;0.0053;0.0014;nan;-1;0.1667;0.0911;0.0267;nan +2020-06-01T00:43:00.0/2020-06-01T00:44:00.0;0.0000;0.0000;0.0000;0.0000;0.0000;0.0000;0.9710;328.6755;14.5239;0.0054;0.0032;0.0006;0.0130;0.0267;0.0053;0.0014;nan;-1;0.1667;0.0911;0.0267;nan +2020-06-01T00:44:00.0/2020-06-01T00:45:00.0;0.0000;0.0000;0.0000;0.0000;0.0000;0.0000;0.9710;328.6672;14.5328;0.0054;0.0032;0.0006;0.0130;0.0267;0.0053;0.0014;nan;-1;0.1667;0.0911;0.0267;nan +2020-06-01T00:45:00.0/2020-06-01T00:46:00.0;0.0000;0.0000;0.0000;0.0000;0.0000;0.0000;0.9710;328.6589;14.5417;0.0054;0.0032;0.0006;0.0130;0.0267;0.0053;0.0014;nan;-1;0.1667;0.0911;0.0267;nan +2020-06-01T00:46:00.0/2020-06-01T00:47:00.0;0.0000;0.0000;0.0000;0.0000;0.0000;0.0000;0.9710;328.6506;14.5506;0.0054;0.0032;0.0006;0.0130;0.0267;0.0053;0.0014;nan;-1;0.1667;0.0911;0.0267;nan +2020-06-01T00:47:00.0/2020-06-01T00:48:00.0;0.0000;0.0000;0.0000;0.0000;0.0000;0.0000;0.9710;328.6423;14.5595;0.0054;0.0032;0.0006;0.0130;0.0267;0.0053;0.0014;nan;-1;0.1667;0.0911;0.0267;nan +2020-06-01T00:48:00.0/2020-06-01T00:49:00.0;0.0000;0.0000;0.0000;0.0000;0.0000;0.0000;0.9710;328.6340;14.5684;0.0054;0.0032;0.0006;0.0130;0.0267;0.0053;0.0014;nan;-1;0.1667;0.0911;0.0267;nan +2020-06-01T00:49:00.0/2020-06-01T00:50:00.0;0.0000;0.0000;0.0000;0.0000;0.0000;0.0000;0.9710;328.6257;14.5774;0.0054;0.0033;0.0006;0.0130;0.0267;0.0053;0.0014;nan;-1;0.1667;0.0911;0.0267;nan +2020-06-01T00:50:00.0/2020-06-01T00:51:00.0;0.0000;0.0000;0.0000;0.0000;0.0000;0.0000;0.9710;328.6174;14.5863;0.0054;0.0033;0.0006;0.0131;0.0267;0.0053;0.0014;nan;-1;0.1667;0.0911;0.0267;nan +2020-06-01T00:51:00.0/2020-06-01T00:52:00.0;0.0000;0.0000;0.0000;0.0000;0.0000;0.0000;0.9710;328.6091;14.5952;0.0054;0.0033;0.0006;0.0131;0.0267;0.0053;0.0014;nan;-1;0.1667;0.0911;0.0267;nan +2020-06-01T00:52:00.0/2020-06-01T00:53:00.0;0.0000;0.0000;0.0000;0.0000;0.0000;0.0000;0.9710;328.6008;14.6041;0.0054;0.0033;0.0006;0.0131;0.0267;0.0053;0.0014;nan;-1;0.1667;0.0911;0.0267;nan +2020-06-01T00:53:00.0/2020-06-01T00:54:00.0;0.0000;0.0000;0.0000;0.0000;0.0000;0.0000;0.9710;328.5925;14.6130;0.0054;0.0033;0.0006;0.0131;0.0267;0.0053;0.0014;nan;-1;0.1667;0.0911;0.0267;nan +2020-06-01T00:54:00.0/2020-06-01T00:55:00.0;0.0000;0.0000;0.0000;0.0000;0.0000;0.0000;0.9710;328.5842;14.6219;0.0054;0.0033;0.0006;0.0131;0.0267;0.0053;0.0014;nan;-1;0.1667;0.0911;0.0267;nan +2020-06-01T00:55:00.0/2020-06-01T00:56:00.0;0.0000;0.0000;0.0000;0.0000;0.0000;0.0000;0.9710;328.5759;14.6308;0.0054;0.0033;0.0006;0.0131;0.0267;0.0053;0.0014;nan;-1;0.1667;0.0911;0.0267;nan +2020-06-01T00:56:00.0/2020-06-01T00:57:00.0;0.0000;0.0000;0.0000;0.0000;0.0000;0.0000;0.9710;328.5676;14.6397;0.0054;0.0033;0.0006;0.0131;0.0267;0.0053;0.0014;nan;-1;0.1667;0.0911;0.0267;nan +2020-06-01T00:57:00.0/2020-06-01T00:58:00.0;0.0000;0.0000;0.0000;0.0000;0.0000;0.0000;0.9710;328.5593;14.6486;0.0054;0.0033;0.0006;0.0131;0.0267;0.0052;0.0014;nan;-1;0.1667;0.0911;0.0267;nan +2020-06-01T00:58:00.0/2020-06-01T00:59:00.0;0.0000;0.0000;0.0000;0.0000;0.0000;0.0000;0.9710;328.5510;14.6575;0.0054;0.0033;0.0006;0.0131;0.0267;0.0052;0.0014;nan;-1;0.1667;0.0911;0.0267;nan +2020-06-01T00:59:00.0/2020-06-01T01:00:00.0;0.0000;0.0000;0.0000;0.0000;0.0000;0.0000;0.9710;328.5427;14.6664;0.0054;0.0033;0.0006;0.0131;0.0267;0.0052;0.0014;nan;-1;0.1667;0.0911;0.0267;nan From 2092f8b3838925177870644ef8694085624cfa0b Mon Sep 17 00:00:00 2001 From: AdamRJensen Date: Tue, 23 Feb 2021 23:06:54 +0100 Subject: [PATCH 06/57] Add test file for monthly data --- pvlib/data/cams_mcclear_monthly.csv | 50 +++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 pvlib/data/cams_mcclear_monthly.csv diff --git a/pvlib/data/cams_mcclear_monthly.csv b/pvlib/data/cams_mcclear_monthly.csv new file mode 100644 index 0000000000..f490cf2792 --- /dev/null +++ b/pvlib/data/cams_mcclear_monthly.csv @@ -0,0 +1,50 @@ +# Coding: utf-8 +# File format version: 2 +# Title: CAMS McClear v3.1 model of clear-sky irradiation. +# Content: A time-series of solar radiation received on horizontal plane and plane always normal to the sun rays at ground level assuming clear sky. +# Calls on the McClear clear-sky model. Returns the global, beam and diffuse irradiations integrated over a selected time step, +# for a selected location (worldwide coverage) and a selected period. +# The research leading to these results has received funding from the European Union within the Copernicus programme. +# Provider: MINES ParisTech (France) +# More information at: http://www.soda-pro.com/web-services/radiation/cams-mcclear +# Date begin (ISO 8601): 2020-01-01T00:00:00.0 +# Date end (ISO 8601): 2021-01-01T00:00:00.0 +# Latitude (positive North, ISO 19115): 55.7906 +# Longitude (positive East, ISO 19115): 12.5251 +# Altitude (m): 39.00 +# Time reference: Universal time (UT) +# +# Encoding partly from D2.8.III.13-14 INSPIRE Data Specification on Atmospheric Conditions and Meteorological Geographical Features - Technical Guidelines (2013-12-10) and CF (Climate and Forecast) metadata (2013-11-11) +# CF Standard Names registry of ObservablePropertyValue +# http://cfconventions.org/Data/cf-standard-names/27/build/cf-standard-name-table.html +# urn:x-inspire:specification:DS-AC-MF:observable-property-name:cf-standard-name:1.6 +# ObservableProperty +# basePhenomenon:"integral_of_surface_downwelling_shortwave_flux_in_air_assuming_clear_sky_wrt_time" +# uom:"Wh m-2" [unit] +# StatisticalMeasure +# statisticalFunction: "sum" +# Summarization (integration) period: 0 year 1 month 0 day 0 h 0 min 0 s +# noValue: nan +# File generated on: 2021-02-23 +# +# Columns: +# 1. Observation period (ISO 8601) +# 2. TOA. Irradiation on horizontal plane at the top of atmosphere (Wh/m2) +# 3. Clear sky GHI. Clear sky global irradiation on horizontal plane at ground level (Wh/m2) +# 4. Clear sky BHI. Clear sky beam irradiation on horizontal plane at ground level (Wh/m2) +# 5. Clear sky DHI. Clear sky diffuse irradiation on horizontal plane at ground level (Wh/m2) +# 6. Clear sky BNI. Clear sky beam irradiation on mobile plane following the sun at normal incidence (Wh/m2) +# +# Observation period;TOA;Clear sky GHI;Clear sky BHI;Clear sky DHI;Clear sky BNI +2020-01-01T00:00:00.0/2020-02-01T00:00:00.0;50168.9961;29424.7578;19492.6445;9932.1123;105764.1953 +2020-02-01T00:00:00.0/2020-03-01T00:00:00.0;91338.5234;59010.3047;40636.3008;18374.0020;140930.5781 +2020-03-01T00:00:00.0/2020-04-01T00:00:00.0;172855.2031;121402.9141;93124.6250;28278.2891;228798.9062 +2020-04-01T00:00:00.0/2020-05-01T00:00:00.0;248215.0312;180546.1406;142470.4688;38075.6680;279122.9375 +2020-05-01T00:00:00.0/2020-06-01T00:00:00.0;323612.6562;236514.2969;192534.5156;43979.7695;333939.2500 +2020-06-01T00:00:00.0/2020-07-01T00:00:00.0;342520.5938;243592.3438;190944.2812;52648.0664;314883.6875 +2020-07-01T00:00:00.0/2020-08-01T00:00:00.0;336242.4375;241529.2656;195424.1250;46105.1484;331152.5625 +2020-08-01T00:00:00.0/2020-09-01T00:00:00.0;277367.4375;194389.0000;150684.6250;43704.3633;278330.5312 +2020-09-01T00:00:00.0/2020-10-01T00:00:00.0;191782.4062;128731.1172;93966.3203;34764.7969;206941.1250 +2020-10-01T00:00:00.0/2020-11-01T00:00:00.0;119269.3047;77265.0312;55354.8828;21910.1484;168171.4062 +2020-11-01T00:00:00.0/2020-12-01T00:00:00.0;59529.1133;35376.3125;23355.3477;12020.9658;110117.4375 +2020-12-01T00:00:00.0/2021-01-01T00:00:00.0;37984.8984;20478.5742;12214.5166;8264.0566;80236.4766 From ff9cece285255497eaa619e60665497e43f5afbf Mon Sep 17 00:00:00 2001 From: AdamRJensen Date: Wed, 24 Feb 2021 00:56:48 +0100 Subject: [PATCH 07/57] Create sub-functions parse and read --- pvlib/iotools/cams.py | 162 +++++++++++++++++++++++++++++++----------- 1 file changed, 122 insertions(+), 40 deletions(-) diff --git a/pvlib/iotools/cams.py b/pvlib/iotools/cams.py index c802420623..450787947a 100644 --- a/pvlib/iotools/cams.py +++ b/pvlib/iotools/cams.py @@ -112,6 +112,9 @@ def get_cams_mcclear(start_date, end_date, latitude, longitude, email, e.g. `sza` becomes `solar_zenith`. See the `pvlib.iotools.cams.MCCLEAR_VARIABLE_MAP` dict for the complete mapping. + See Also + -------- + pvlib.iotools.read_cams_mcclear, pvlib.iotools.parse_cams_mcclear References ---------- @@ -165,43 +168,122 @@ def get_cams_mcclear(start_date, end_date, latitude, longitude, email, # Check if returned file is a csv data file elif res.headers['Content-Type'] == 'application/csv': - data = pd.read_csv(io.StringIO(res.content.decode('utf-8')), sep=';', - comment='#', header=None, names=names) - - obs_period = data['Observation period'].str.split('/') - - # Set index as the start observation time (left) and localize to UTC - if (label == 'left') | ((label is None) & (time_step != '1M')): - data.index = pd.to_datetime(obs_period.str[0], utc=True) - # Set index as the stop observation time (right) and localize to UTC - elif (label == 'right') | ((label is None) & (time_step == '1M')): - data.index = pd.to_datetime(obs_period.str[1], utc=True) - - data.index.name = None # Set index name to None - - # Change index for '1d' and '1M' to be date and not datetime - if time_step == '1d': - data.index = data.index.date - elif (time_step == '1M') & (label is not None): - data.index = data.index.date - # For monthly data with 'right' label, the index should be the last - # date of the month and not the first date of the following month - elif (time_step == '1M') & (time_step != 'left'): - data.index = data.index.date - pd.Timestamp(days=1) - - if not integrated: # Convert from Wh/m2 to W/m2 - integrated_cols = MCCLEAR_COLUMNS[1:6] - - if time_step == '1M': - time_delta = (pd.to_datetime(obs_period.str[1]) - - pd.to_datetime(obs_period.str[0])) - hours = time_delta.dt.total_seconds()/60/60 - data[integrated_cols] = data[integrated_cols] / hours - else: - data[integrated_cols] = (data[integrated_cols] / - TIME_STEPS_HOURS[time_step]) - - if map_variables: - data = data.rename(columns=MCCLEAR_VARIABLE_MAP) - - return data + fbuf = io.StringIO(res.content.decode('utf-8')) + data, meta = parse_cams_mcclear(fbuf, integrated=integrated, + label=label, + map_variables=map_variables) + return data, meta + else: + print('Error in recognizing file content occurred!') + + +def parse_cams_mcclear(fbuf, integrated=False, label=None, map_variables=True): + """ + Parse a CAMS McClear file. CAMS McClear is described in [1]_. + + + See Also + -------- + pvlib.iotools.read_cams_mcclear, pvlib.iotools.get_cams_mcclear + + References + ---------- + .. [1] `CAMS McClear Service Info + `_ + """ + meta = {} + # Initial lines of the file contain meta-data, which all start with # + while True: + line = fbuf.readline().rstrip('\n') + if not line.startswith('#'): + break + elif ': ' in line: + meta[line.split(': ')[0].lstrip('# ')] = line.split(': ')[1] + # The last line of the meta-data section contains the column names + names = line.lstrip('# ').split(';') + + # Convert the latitude, longitude, and altitude from strings to floats + meta['Latitude (positive North, ISO 19115)'] = \ + float(meta['Latitude (positive North, ISO 19115)']) + meta['Longitude (positive East, ISO 19115)'] = \ + float(meta['Longitude (positive East, ISO 19115)']) + meta['Altitude (m)'] = float(meta['Altitude (m)']) + + # Determine the time_step from the meta-data dictionary + time_step_dict = {'0 year 0 month 0 day 0 h 1 min 0 s': '1min', + '0 year 0 month 0 day 0 h 15 min 0 s': '15min', + '0 year 0 month 0 day 1 h 0 min 0 s': '1h', + '0 year 0 month 1 day 0 h 1 min 0 s': '1d', + '0 year 1 month 0 day 0 h 0 min 0 s': '1M'} + time_step = time_step_dict[meta['Summarization (integration) period']] + + data = pd.read_csv(fbuf, sep=';', comment='#', header=None, names=names) + + obs_period = data['Observation period'].str.split('/') + + # Set index as the start observation time (left) and localize to UTC + if (label == 'left') | ((label is None) & (time_step != '1M')): + data.index = pd.to_datetime(obs_period.str[0], utc=True) + # Set index as the stop observation time (right) and localize to UTC + elif (label == 'right') | ((label is None) & (time_step == '1M')): + data.index = pd.to_datetime(obs_period.str[1], utc=True) + + data.index.name = 'time' # Set index name to None + + # Change index for '1d' and '1M' to be date and not datetime + if time_step == '1d': + data.index = data.index.date + elif (time_step == '1M') & (label is not None): + data.index = data.index.date + # For monthly data with 'right' label, the index should be the last + # date of the month and not the first date of the following month + elif (time_step == '1M') & (time_step != 'left'): + data.index = data.index.date - pd.Timestamp(days=1) + + if not integrated: # Convert from Wh/m2 to W/m2 + integrated_cols = MCCLEAR_COLUMNS[1:6] + + if time_step == '1M': + time_delta = (pd.to_datetime(obs_period.str[1]) + - pd.to_datetime(obs_period.str[0])) + hours = time_delta.dt.total_seconds()/60/60 + data[integrated_cols] = data[integrated_cols] / hours + else: + data[integrated_cols] = (data[integrated_cols] / + TIME_STEPS_HOURS[time_step]) + + if map_variables: + data = data.rename(columns=MCCLEAR_VARIABLE_MAP) + + return data, meta + + +def read_cams_mcclear(filename): + """ + Read a CAMS McClear file. CAMS McClear is described in [1]_. + + Parameters + ---------- + filename: str + Filename of a file containing data to read. + + Returns + ------- + data : pandas.DataFrame + timeseries data from CAMS McClear + meta : dict + metadata from CAMS McClear, see + :func:`pvlib.iotools.parse_cams_mcclear` for fields + + See Also + -------- + pvlib.iotools.parse_cams_mcclear, pvlib.iotools.get_cams_mcclear + + References + ---------- + .. [1] `CAMS McClear Service Info + `_ + """ + with open(str(filename), 'r') as fbuf: + content = parse_cams_mcclear(fbuf) + return content From 0c282994f9679152c0701aef503fdb3a6e54832b Mon Sep 17 00:00:00 2001 From: AdamRJensen Date: Wed, 24 Feb 2021 00:58:29 +0100 Subject: [PATCH 08/57] Fixed stickler --- pvlib/iotools/cams.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pvlib/iotools/cams.py b/pvlib/iotools/cams.py index 450787947a..d556800cd3 100644 --- a/pvlib/iotools/cams.py +++ b/pvlib/iotools/cams.py @@ -181,7 +181,6 @@ def parse_cams_mcclear(fbuf, integrated=False, label=None, map_variables=True): """ Parse a CAMS McClear file. CAMS McClear is described in [1]_. - See Also -------- pvlib.iotools.read_cams_mcclear, pvlib.iotools.get_cams_mcclear From 75e575f9236e51c7faae02b89ee73689aa480613 Mon Sep 17 00:00:00 2001 From: AdamRJensen Date: Thu, 25 Feb 2021 22:40:23 +0100 Subject: [PATCH 09/57] Update constants names --- pvlib/iotools/cams.py | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/pvlib/iotools/cams.py b/pvlib/iotools/cams.py index d556800cd3..c591a6a658 100644 --- a/pvlib/iotools/cams.py +++ b/pvlib/iotools/cams.py @@ -28,10 +28,16 @@ # Dictionary mapping Python time steps to CAMS time step format -TIME_STEPS = {'1min': 'PT01M', '15min': 'PT15M', '1h': 'PT01H', '1d': 'P01D', - '1M': 'P01M'} +TIME_STEPS_MAP = {'1min': 'PT01M', '15min': 'PT15M', '1h': 'PT01H', '1d': 'P01D', + '1M': 'P01M'} -TIME_STEPS_HOURS = {'1min': 1/60, '15min': 15/60, '1h': 1, '1d': 24} +TIME_STEPS_IN_HOURS = {'1min': 1/60, '15min': 15/60, '1h': 1, '1d': 24} + +SUMMATION_PERIOD_TO_TIME_STEP = {'0 year 0 month 0 day 0 h 1 min 0 s': '1min', + '0 year 0 month 0 day 0 h 15 min 0 s': '15min', + '0 year 0 month 0 day 1 h 0 min 0 s': '1h', + '0 year 0 month 1 day 0 h 1 min 0 s': '1d', + '0 year 1 month 0 day 0 h 0 min 0 s': '1M'} def get_cams_mcclear(start_date, end_date, latitude, longitude, email, @@ -124,8 +130,8 @@ def get_cams_mcclear(start_date, end_date, latitude, longitude, email, `_ """ - if time_step in TIME_STEPS.keys(): - time_step_str = TIME_STEPS[time_step] + if time_step in TIME_STEPS_MAP.keys(): + time_step_str = TIME_STEPS_MAP[time_step] else: print('WARNING: time step not recognized, 1 hour time step used!') time_step_str = 'PT01H' @@ -174,7 +180,7 @@ def get_cams_mcclear(start_date, end_date, latitude, longitude, email, map_variables=map_variables) return data, meta else: - print('Error in recognizing file content occurred!') + print('Error in recognizing the file content occurred!') def parse_cams_mcclear(fbuf, integrated=False, label=None, map_variables=True): @@ -209,12 +215,9 @@ def parse_cams_mcclear(fbuf, integrated=False, label=None, map_variables=True): meta['Altitude (m)'] = float(meta['Altitude (m)']) # Determine the time_step from the meta-data dictionary - time_step_dict = {'0 year 0 month 0 day 0 h 1 min 0 s': '1min', - '0 year 0 month 0 day 0 h 15 min 0 s': '15min', - '0 year 0 month 0 day 1 h 0 min 0 s': '1h', - '0 year 0 month 1 day 0 h 1 min 0 s': '1d', - '0 year 1 month 0 day 0 h 0 min 0 s': '1M'} - time_step = time_step_dict[meta['Summarization (integration) period']] + time_step = SUMMATION_PERIOD_TO_TIME_STEP[ + meta['Summarization (integration) period']] + meta['time_step'] = time_step data = pd.read_csv(fbuf, sep=';', comment='#', header=None, names=names) @@ -249,7 +252,7 @@ def parse_cams_mcclear(fbuf, integrated=False, label=None, map_variables=True): data[integrated_cols] = data[integrated_cols] / hours else: data[integrated_cols] = (data[integrated_cols] / - TIME_STEPS_HOURS[time_step]) + TIME_STEPS_IN_HOURS[time_step]) if map_variables: data = data.rename(columns=MCCLEAR_VARIABLE_MAP) From 1f2ec30eda1b907fb1bfb010e6f80cf676a8d555 Mon Sep 17 00:00:00 2001 From: AdamRJensen Date: Thu, 25 Feb 2021 23:46:30 +0100 Subject: [PATCH 10/57] Fixed monthly integration of values --- pvlib/iotools/cams.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/pvlib/iotools/cams.py b/pvlib/iotools/cams.py index c591a6a658..ec4905a97a 100644 --- a/pvlib/iotools/cams.py +++ b/pvlib/iotools/cams.py @@ -36,7 +36,7 @@ SUMMATION_PERIOD_TO_TIME_STEP = {'0 year 0 month 0 day 0 h 1 min 0 s': '1min', '0 year 0 month 0 day 0 h 15 min 0 s': '15min', '0 year 0 month 0 day 1 h 0 min 0 s': '1h', - '0 year 0 month 1 day 0 h 1 min 0 s': '1d', + '0 year 0 month 1 day 0 h 0 min 0 s': '1d', '0 year 1 month 0 day 0 h 0 min 0 s': '1M'} @@ -227,20 +227,19 @@ def parse_cams_mcclear(fbuf, integrated=False, label=None, map_variables=True): if (label == 'left') | ((label is None) & (time_step != '1M')): data.index = pd.to_datetime(obs_period.str[0], utc=True) # Set index as the stop observation time (right) and localize to UTC + # default label for monthly data is 'right' following Pandas' convention elif (label == 'right') | ((label is None) & (time_step == '1M')): data.index = pd.to_datetime(obs_period.str[1], utc=True) data.index.name = 'time' # Set index name to None - # Change index for '1d' and '1M' to be date and not datetime - if time_step == '1d': - data.index = data.index.date - elif (time_step == '1M') & (label is not None): - data.index = data.index.date + # Change index for time_step '1d' and '1M' to be date and not datetime + if (time_step == '1d') | (time_step == '1M'): + data.index = pd.DatetimeIndex(data.index.date) # For monthly data with 'right' label, the index should be the last # date of the month and not the first date of the following month - elif (time_step == '1M') & (time_step != 'left'): - data.index = data.index.date - pd.Timestamp(days=1) + if (time_step == '1M') & (label != 'left'): + data.index = data.index - pd.Timedelta(days=1) if not integrated: # Convert from Wh/m2 to W/m2 integrated_cols = MCCLEAR_COLUMNS[1:6] @@ -249,7 +248,8 @@ def parse_cams_mcclear(fbuf, integrated=False, label=None, map_variables=True): time_delta = (pd.to_datetime(obs_period.str[1]) - pd.to_datetime(obs_period.str[0])) hours = time_delta.dt.total_seconds()/60/60 - data[integrated_cols] = data[integrated_cols] / hours + data[integrated_cols] = data[integrated_cols].\ + divide(hours.tolist(), axis='rows') else: data[integrated_cols] = (data[integrated_cols] / TIME_STEPS_IN_HOURS[time_step]) From 641fc978b3dc5982c2a328d63a016f24c0ba0537 Mon Sep 17 00:00:00 2001 From: AdamRJensen Date: Fri, 26 Feb 2021 00:00:20 +0100 Subject: [PATCH 11/57] Improvement to meta-data parsing --- pvlib/iotools/cams.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/pvlib/iotools/cams.py b/pvlib/iotools/cams.py index ec4905a97a..ce9bc352ef 100644 --- a/pvlib/iotools/cams.py +++ b/pvlib/iotools/cams.py @@ -200,12 +200,12 @@ def parse_cams_mcclear(fbuf, integrated=False, label=None, map_variables=True): # Initial lines of the file contain meta-data, which all start with # while True: line = fbuf.readline().rstrip('\n') - if not line.startswith('#'): - break + if line.startswith('# Observation period'): + # The last line of the meta-data section contains the column names + names = line.lstrip('# ').split(';') + break # End of meta-data section has been reached elif ': ' in line: meta[line.split(': ')[0].lstrip('# ')] = line.split(': ')[1] - # The last line of the meta-data section contains the column names - names = line.lstrip('# ').split(';') # Convert the latitude, longitude, and altitude from strings to floats meta['Latitude (positive North, ISO 19115)'] = \ @@ -213,6 +213,7 @@ def parse_cams_mcclear(fbuf, integrated=False, label=None, map_variables=True): meta['Longitude (positive East, ISO 19115)'] = \ float(meta['Longitude (positive East, ISO 19115)']) meta['Altitude (m)'] = float(meta['Altitude (m)']) + meta['Clear sky radiation unit'] = {True:'Wh/m2', False:'W/m2'}[integrated] # Determine the time_step from the meta-data dictionary time_step = SUMMATION_PERIOD_TO_TIME_STEP[ From 527fa1cf21307bcc0d2dc54585def4ca2626656f Mon Sep 17 00:00:00 2001 From: AdamRJensen Date: Fri, 26 Feb 2021 14:33:42 +0100 Subject: [PATCH 12/57] Update test file to be during the day --- pvlib/data/cams_mcclear_1min_verbose.csv | 122 +++++++++++------------ 1 file changed, 61 insertions(+), 61 deletions(-) diff --git a/pvlib/data/cams_mcclear_1min_verbose.csv b/pvlib/data/cams_mcclear_1min_verbose.csv index c57d7cb3ad..9d192d9dfe 100644 --- a/pvlib/data/cams_mcclear_1min_verbose.csv +++ b/pvlib/data/cams_mcclear_1min_verbose.csv @@ -26,7 +26,7 @@ # statisticalFunction: "sum" # Summarization (integration) period: 0 year 0 month 0 day 0 h 1 min 0 s # noValue: nan -# File generated on: 2021-02-23 +# File generated on: 2021-02-26 # # Columns: # 1. Observation period (ISO 8601) @@ -54,63 +54,63 @@ #23. albedo. Ground albedo # # Observation period;TOA;Clear sky GHI;Clear sky BHI;Clear sky DHI;Clear sky BNI;sza;summer/winter split;tco3;tcwv;AOD BC;AOD DU;AOD SS;AOD OR;AOD SU;AOD NI;AOD AM;alpha;Aerosol type;fiso;fvol;fgeo;albedo -2020-06-01T00:00:00.0/2020-06-01T00:01:00.0;0.0000;0.0000;0.0000;0.0000;0.0000;0.0000;0.9709;329.0324;14.1409;0.0053;0.0031;0.0006;0.0127;0.0268;0.0056;0.0015;nan;-1;0.1667;0.0911;0.0267;nan -2020-06-01T00:01:00.0/2020-06-01T00:02:00.0;0.0000;0.0000;0.0000;0.0000;0.0000;0.0000;0.9709;329.0241;14.1498;0.0053;0.0031;0.0006;0.0127;0.0268;0.0056;0.0015;nan;-1;0.1667;0.0911;0.0267;nan -2020-06-01T00:02:00.0/2020-06-01T00:03:00.0;0.0000;0.0000;0.0000;0.0000;0.0000;0.0000;0.9709;329.0158;14.1587;0.0053;0.0031;0.0006;0.0127;0.0268;0.0056;0.0015;nan;-1;0.1667;0.0911;0.0267;nan -2020-06-01T00:03:00.0/2020-06-01T00:04:00.0;0.0000;0.0000;0.0000;0.0000;0.0000;0.0000;0.9709;329.0075;14.1676;0.0053;0.0031;0.0006;0.0127;0.0268;0.0056;0.0015;nan;-1;0.1667;0.0911;0.0267;nan -2020-06-01T00:04:00.0/2020-06-01T00:05:00.0;0.0000;0.0000;0.0000;0.0000;0.0000;0.0000;0.9709;328.9992;14.1765;0.0053;0.0031;0.0006;0.0127;0.0268;0.0055;0.0015;nan;-1;0.1667;0.0911;0.0267;nan -2020-06-01T00:05:00.0/2020-06-01T00:06:00.0;0.0000;0.0000;0.0000;0.0000;0.0000;0.0000;0.9709;328.9909;14.1854;0.0053;0.0031;0.0006;0.0127;0.0268;0.0055;0.0014;nan;-1;0.1667;0.0911;0.0267;nan -2020-06-01T00:06:00.0/2020-06-01T00:07:00.0;0.0000;0.0000;0.0000;0.0000;0.0000;0.0000;0.9709;328.9826;14.1943;0.0053;0.0031;0.0006;0.0127;0.0268;0.0055;0.0014;nan;-1;0.1667;0.0911;0.0267;nan -2020-06-01T00:07:00.0/2020-06-01T00:08:00.0;0.0000;0.0000;0.0000;0.0000;0.0000;0.0000;0.9709;328.9743;14.2032;0.0053;0.0031;0.0006;0.0127;0.0268;0.0055;0.0014;nan;-1;0.1667;0.0911;0.0267;nan -2020-06-01T00:08:00.0/2020-06-01T00:09:00.0;0.0000;0.0000;0.0000;0.0000;0.0000;0.0000;0.9709;328.9660;14.2121;0.0053;0.0031;0.0006;0.0127;0.0268;0.0055;0.0014;nan;-1;0.1667;0.0911;0.0267;nan -2020-06-01T00:09:00.0/2020-06-01T00:10:00.0;0.0000;0.0000;0.0000;0.0000;0.0000;0.0000;0.9709;328.9577;14.2210;0.0053;0.0031;0.0006;0.0128;0.0268;0.0055;0.0014;nan;-1;0.1667;0.0911;0.0267;nan -2020-06-01T00:10:00.0/2020-06-01T00:11:00.0;0.0000;0.0000;0.0000;0.0000;0.0000;0.0000;0.9709;328.9494;14.2299;0.0053;0.0031;0.0006;0.0128;0.0268;0.0055;0.0014;nan;-1;0.1667;0.0911;0.0267;nan -2020-06-01T00:11:00.0/2020-06-01T00:12:00.0;0.0000;0.0000;0.0000;0.0000;0.0000;0.0000;0.9709;328.9411;14.2388;0.0053;0.0031;0.0006;0.0128;0.0268;0.0055;0.0014;nan;-1;0.1667;0.0911;0.0267;nan -2020-06-01T00:12:00.0/2020-06-01T00:13:00.0;0.0000;0.0000;0.0000;0.0000;0.0000;0.0000;0.9709;328.9328;14.2478;0.0053;0.0031;0.0006;0.0128;0.0268;0.0055;0.0014;nan;-1;0.1667;0.0911;0.0267;nan -2020-06-01T00:13:00.0/2020-06-01T00:14:00.0;0.0000;0.0000;0.0000;0.0000;0.0000;0.0000;0.9709;328.9245;14.2567;0.0053;0.0031;0.0006;0.0128;0.0268;0.0055;0.0014;nan;-1;0.1667;0.0911;0.0267;nan -2020-06-01T00:14:00.0/2020-06-01T00:15:00.0;0.0000;0.0000;0.0000;0.0000;0.0000;0.0000;0.9709;328.9162;14.2656;0.0053;0.0031;0.0006;0.0128;0.0268;0.0055;0.0014;nan;-1;0.1667;0.0911;0.0267;nan -2020-06-01T00:15:00.0/2020-06-01T00:16:00.0;0.0000;0.0000;0.0000;0.0000;0.0000;0.0000;0.9709;328.9079;14.2745;0.0053;0.0031;0.0006;0.0128;0.0268;0.0055;0.0014;nan;-1;0.1667;0.0911;0.0267;nan -2020-06-01T00:16:00.0/2020-06-01T00:17:00.0;0.0000;0.0000;0.0000;0.0000;0.0000;0.0000;0.9710;328.8996;14.2834;0.0053;0.0031;0.0006;0.0128;0.0268;0.0055;0.0014;nan;-1;0.1667;0.0911;0.0267;nan -2020-06-01T00:17:00.0/2020-06-01T00:18:00.0;0.0000;0.0000;0.0000;0.0000;0.0000;0.0000;0.9710;328.8913;14.2923;0.0053;0.0031;0.0006;0.0128;0.0268;0.0055;0.0014;nan;-1;0.1667;0.0911;0.0267;nan -2020-06-01T00:18:00.0/2020-06-01T00:19:00.0;0.0000;0.0000;0.0000;0.0000;0.0000;0.0000;0.9710;328.8830;14.3012;0.0053;0.0031;0.0006;0.0128;0.0268;0.0055;0.0014;nan;-1;0.1667;0.0911;0.0267;nan -2020-06-01T00:19:00.0/2020-06-01T00:20:00.0;0.0000;0.0000;0.0000;0.0000;0.0000;0.0000;0.9710;328.8747;14.3101;0.0053;0.0031;0.0006;0.0128;0.0268;0.0055;0.0014;nan;-1;0.1667;0.0911;0.0267;nan -2020-06-01T00:20:00.0/2020-06-01T00:21:00.0;0.0000;0.0000;0.0000;0.0000;0.0000;0.0000;0.9710;328.8664;14.3190;0.0053;0.0031;0.0006;0.0128;0.0268;0.0055;0.0014;nan;-1;0.1667;0.0911;0.0267;nan -2020-06-01T00:21:00.0/2020-06-01T00:22:00.0;0.0000;0.0000;0.0000;0.0000;0.0000;0.0000;0.9710;328.8581;14.3279;0.0054;0.0032;0.0006;0.0128;0.0268;0.0055;0.0014;nan;-1;0.1667;0.0911;0.0267;nan -2020-06-01T00:22:00.0/2020-06-01T00:23:00.0;0.0000;0.0000;0.0000;0.0000;0.0000;0.0000;0.9710;328.8498;14.3368;0.0054;0.0032;0.0006;0.0128;0.0268;0.0054;0.0014;nan;-1;0.1667;0.0911;0.0267;nan -2020-06-01T00:23:00.0/2020-06-01T00:24:00.0;0.0000;0.0000;0.0000;0.0000;0.0000;0.0000;0.9710;328.8415;14.3457;0.0054;0.0032;0.0006;0.0129;0.0268;0.0054;0.0014;nan;-1;0.1667;0.0911;0.0267;nan -2020-06-01T00:24:00.0/2020-06-01T00:25:00.0;0.0000;0.0000;0.0000;0.0000;0.0000;0.0000;0.9710;328.8332;14.3547;0.0054;0.0032;0.0006;0.0129;0.0268;0.0054;0.0014;nan;-1;0.1667;0.0911;0.0267;nan -2020-06-01T00:25:00.0/2020-06-01T00:26:00.0;0.0000;0.0000;0.0000;0.0000;0.0000;0.0000;0.9710;328.8249;14.3636;0.0054;0.0032;0.0006;0.0129;0.0268;0.0054;0.0014;nan;-1;0.1667;0.0911;0.0267;nan -2020-06-01T00:26:00.0/2020-06-01T00:27:00.0;0.0000;0.0000;0.0000;0.0000;0.0000;0.0000;0.9710;328.8166;14.3725;0.0054;0.0032;0.0006;0.0129;0.0268;0.0054;0.0014;nan;-1;0.1667;0.0911;0.0267;nan -2020-06-01T00:27:00.0/2020-06-01T00:28:00.0;0.0000;0.0000;0.0000;0.0000;0.0000;0.0000;0.9710;328.8083;14.3814;0.0054;0.0032;0.0006;0.0129;0.0268;0.0054;0.0014;nan;-1;0.1667;0.0911;0.0267;nan -2020-06-01T00:28:00.0/2020-06-01T00:29:00.0;0.0000;0.0000;0.0000;0.0000;0.0000;0.0000;0.9710;328.8000;14.3903;0.0054;0.0032;0.0006;0.0129;0.0268;0.0054;0.0014;nan;-1;0.1667;0.0911;0.0267;nan -2020-06-01T00:29:00.0/2020-06-01T00:30:00.0;0.0000;0.0000;0.0000;0.0000;0.0000;0.0000;0.9710;328.7917;14.3992;0.0054;0.0032;0.0006;0.0129;0.0268;0.0054;0.0014;nan;-1;0.1667;0.0911;0.0267;nan -2020-06-01T00:30:00.0/2020-06-01T00:31:00.0;0.0000;0.0000;0.0000;0.0000;0.0000;0.0000;0.9710;328.7834;14.4081;0.0054;0.0032;0.0006;0.0129;0.0268;0.0054;0.0014;nan;-1;0.1667;0.0911;0.0267;nan -2020-06-01T00:31:00.0/2020-06-01T00:32:00.0;0.0000;0.0000;0.0000;0.0000;0.0000;0.0000;0.9710;328.7751;14.4170;0.0054;0.0032;0.0006;0.0129;0.0268;0.0054;0.0014;nan;-1;0.1667;0.0911;0.0267;nan -2020-06-01T00:32:00.0/2020-06-01T00:33:00.0;0.0000;0.0000;0.0000;0.0000;0.0000;0.0000;0.9710;328.7668;14.4259;0.0054;0.0032;0.0006;0.0129;0.0268;0.0054;0.0014;nan;-1;0.1667;0.0911;0.0267;nan -2020-06-01T00:33:00.0/2020-06-01T00:34:00.0;0.0000;0.0000;0.0000;0.0000;0.0000;0.0000;0.9710;328.7585;14.4348;0.0054;0.0032;0.0006;0.0129;0.0268;0.0054;0.0014;nan;-1;0.1667;0.0911;0.0267;nan -2020-06-01T00:34:00.0/2020-06-01T00:35:00.0;0.0000;0.0000;0.0000;0.0000;0.0000;0.0000;0.9710;328.7502;14.4437;0.0054;0.0032;0.0006;0.0129;0.0268;0.0054;0.0014;nan;-1;0.1667;0.0911;0.0267;nan -2020-06-01T00:35:00.0/2020-06-01T00:36:00.0;0.0000;0.0000;0.0000;0.0000;0.0000;0.0000;0.9710;328.7419;14.4526;0.0054;0.0032;0.0006;0.0129;0.0268;0.0054;0.0014;nan;-1;0.1667;0.0911;0.0267;nan -2020-06-01T00:36:00.0/2020-06-01T00:37:00.0;0.0000;0.0000;0.0000;0.0000;0.0000;0.0000;0.9710;328.7336;14.4615;0.0054;0.0032;0.0006;0.0130;0.0268;0.0054;0.0014;nan;-1;0.1667;0.0911;0.0267;nan -2020-06-01T00:37:00.0/2020-06-01T00:38:00.0;0.0000;0.0000;0.0000;0.0000;0.0000;0.0000;0.9710;328.7253;14.4705;0.0054;0.0032;0.0006;0.0130;0.0268;0.0054;0.0014;nan;-1;0.1667;0.0911;0.0267;nan -2020-06-01T00:38:00.0/2020-06-01T00:39:00.0;0.0000;0.0000;0.0000;0.0000;0.0000;0.0000;0.9710;328.7170;14.4794;0.0054;0.0032;0.0006;0.0130;0.0267;0.0054;0.0014;nan;-1;0.1667;0.0911;0.0267;nan -2020-06-01T00:39:00.0/2020-06-01T00:40:00.0;0.0000;0.0000;0.0000;0.0000;0.0000;0.0000;0.9710;328.7087;14.4883;0.0054;0.0032;0.0006;0.0130;0.0267;0.0053;0.0014;nan;-1;0.1667;0.0911;0.0267;nan -2020-06-01T00:40:00.0/2020-06-01T00:41:00.0;0.0000;0.0000;0.0000;0.0000;0.0000;0.0000;0.9710;328.7004;14.4972;0.0054;0.0032;0.0006;0.0130;0.0267;0.0053;0.0014;nan;-1;0.1667;0.0911;0.0267;nan -2020-06-01T00:41:00.0/2020-06-01T00:42:00.0;0.0000;0.0000;0.0000;0.0000;0.0000;0.0000;0.9710;328.6921;14.5061;0.0054;0.0032;0.0006;0.0130;0.0267;0.0053;0.0014;nan;-1;0.1667;0.0911;0.0267;nan -2020-06-01T00:42:00.0/2020-06-01T00:43:00.0;0.0000;0.0000;0.0000;0.0000;0.0000;0.0000;0.9710;328.6838;14.5150;0.0054;0.0032;0.0006;0.0130;0.0267;0.0053;0.0014;nan;-1;0.1667;0.0911;0.0267;nan -2020-06-01T00:43:00.0/2020-06-01T00:44:00.0;0.0000;0.0000;0.0000;0.0000;0.0000;0.0000;0.9710;328.6755;14.5239;0.0054;0.0032;0.0006;0.0130;0.0267;0.0053;0.0014;nan;-1;0.1667;0.0911;0.0267;nan -2020-06-01T00:44:00.0/2020-06-01T00:45:00.0;0.0000;0.0000;0.0000;0.0000;0.0000;0.0000;0.9710;328.6672;14.5328;0.0054;0.0032;0.0006;0.0130;0.0267;0.0053;0.0014;nan;-1;0.1667;0.0911;0.0267;nan -2020-06-01T00:45:00.0/2020-06-01T00:46:00.0;0.0000;0.0000;0.0000;0.0000;0.0000;0.0000;0.9710;328.6589;14.5417;0.0054;0.0032;0.0006;0.0130;0.0267;0.0053;0.0014;nan;-1;0.1667;0.0911;0.0267;nan -2020-06-01T00:46:00.0/2020-06-01T00:47:00.0;0.0000;0.0000;0.0000;0.0000;0.0000;0.0000;0.9710;328.6506;14.5506;0.0054;0.0032;0.0006;0.0130;0.0267;0.0053;0.0014;nan;-1;0.1667;0.0911;0.0267;nan -2020-06-01T00:47:00.0/2020-06-01T00:48:00.0;0.0000;0.0000;0.0000;0.0000;0.0000;0.0000;0.9710;328.6423;14.5595;0.0054;0.0032;0.0006;0.0130;0.0267;0.0053;0.0014;nan;-1;0.1667;0.0911;0.0267;nan -2020-06-01T00:48:00.0/2020-06-01T00:49:00.0;0.0000;0.0000;0.0000;0.0000;0.0000;0.0000;0.9710;328.6340;14.5684;0.0054;0.0032;0.0006;0.0130;0.0267;0.0053;0.0014;nan;-1;0.1667;0.0911;0.0267;nan -2020-06-01T00:49:00.0/2020-06-01T00:50:00.0;0.0000;0.0000;0.0000;0.0000;0.0000;0.0000;0.9710;328.6257;14.5774;0.0054;0.0033;0.0006;0.0130;0.0267;0.0053;0.0014;nan;-1;0.1667;0.0911;0.0267;nan -2020-06-01T00:50:00.0/2020-06-01T00:51:00.0;0.0000;0.0000;0.0000;0.0000;0.0000;0.0000;0.9710;328.6174;14.5863;0.0054;0.0033;0.0006;0.0131;0.0267;0.0053;0.0014;nan;-1;0.1667;0.0911;0.0267;nan -2020-06-01T00:51:00.0/2020-06-01T00:52:00.0;0.0000;0.0000;0.0000;0.0000;0.0000;0.0000;0.9710;328.6091;14.5952;0.0054;0.0033;0.0006;0.0131;0.0267;0.0053;0.0014;nan;-1;0.1667;0.0911;0.0267;nan -2020-06-01T00:52:00.0/2020-06-01T00:53:00.0;0.0000;0.0000;0.0000;0.0000;0.0000;0.0000;0.9710;328.6008;14.6041;0.0054;0.0033;0.0006;0.0131;0.0267;0.0053;0.0014;nan;-1;0.1667;0.0911;0.0267;nan -2020-06-01T00:53:00.0/2020-06-01T00:54:00.0;0.0000;0.0000;0.0000;0.0000;0.0000;0.0000;0.9710;328.5925;14.6130;0.0054;0.0033;0.0006;0.0131;0.0267;0.0053;0.0014;nan;-1;0.1667;0.0911;0.0267;nan -2020-06-01T00:54:00.0/2020-06-01T00:55:00.0;0.0000;0.0000;0.0000;0.0000;0.0000;0.0000;0.9710;328.5842;14.6219;0.0054;0.0033;0.0006;0.0131;0.0267;0.0053;0.0014;nan;-1;0.1667;0.0911;0.0267;nan -2020-06-01T00:55:00.0/2020-06-01T00:56:00.0;0.0000;0.0000;0.0000;0.0000;0.0000;0.0000;0.9710;328.5759;14.6308;0.0054;0.0033;0.0006;0.0131;0.0267;0.0053;0.0014;nan;-1;0.1667;0.0911;0.0267;nan -2020-06-01T00:56:00.0/2020-06-01T00:57:00.0;0.0000;0.0000;0.0000;0.0000;0.0000;0.0000;0.9710;328.5676;14.6397;0.0054;0.0033;0.0006;0.0131;0.0267;0.0053;0.0014;nan;-1;0.1667;0.0911;0.0267;nan -2020-06-01T00:57:00.0/2020-06-01T00:58:00.0;0.0000;0.0000;0.0000;0.0000;0.0000;0.0000;0.9710;328.5593;14.6486;0.0054;0.0033;0.0006;0.0131;0.0267;0.0052;0.0014;nan;-1;0.1667;0.0911;0.0267;nan -2020-06-01T00:58:00.0/2020-06-01T00:59:00.0;0.0000;0.0000;0.0000;0.0000;0.0000;0.0000;0.9710;328.5510;14.6575;0.0054;0.0033;0.0006;0.0131;0.0267;0.0052;0.0014;nan;-1;0.1667;0.0911;0.0267;nan -2020-06-01T00:59:00.0/2020-06-01T01:00:00.0;0.0000;0.0000;0.0000;0.0000;0.0000;0.0000;0.9710;328.5427;14.6664;0.0054;0.0033;0.0006;0.0131;0.0267;0.0052;0.0014;nan;-1;0.1667;0.0911;0.0267;nan +2020-06-01T12:00:00.0/2020-06-01T12:01:00.0;18.0699;14.1417;12.5594;1.5823;15.3380;35.0308;0.9723;341.0221;17.7962;0.0065;0.0067;0.0008;0.0215;0.0252;0.0087;0.0022;nan;-1;0.1668;0.0912;0.0267;0.1359 +2020-06-01T12:01:00.0/2020-06-01T12:02:00.0;18.0584;14.1311;12.5484;1.5827;15.3343;35.0828;0.9723;341.0223;17.8020;0.0065;0.0067;0.0008;0.0215;0.0253;0.0087;0.0022;nan;-1;0.1668;0.0912;0.0267;0.1359 +2020-06-01T12:02:00.0/2020-06-01T12:03:00.0;18.0467;14.1204;12.5372;1.5831;15.3306;35.1357;0.9723;341.0224;17.8079;0.0065;0.0067;0.0008;0.0216;0.0253;0.0087;0.0022;nan;-1;0.1668;0.0912;0.0267;0.1359 +2020-06-01T12:03:00.0/2020-06-01T12:04:00.0;18.0348;14.1094;12.5259;1.5835;15.3269;35.1896;0.9723;341.0226;17.8137;0.0065;0.0067;0.0008;0.0217;0.0253;0.0087;0.0022;nan;-1;0.1668;0.0912;0.0267;0.1359 +2020-06-01T12:04:00.0/2020-06-01T12:05:00.0;18.0226;14.0983;12.5144;1.5839;15.3231;35.2442;0.9723;341.0228;17.8196;0.0064;0.0067;0.0008;0.0218;0.0254;0.0087;0.0022;nan;-1;0.1668;0.0912;0.0267;0.1360 +2020-06-01T12:05:00.0/2020-06-01T12:06:00.0;18.0103;14.0869;12.5026;1.5843;15.3193;35.2998;0.9723;341.0229;17.8254;0.0064;0.0067;0.0008;0.0218;0.0254;0.0087;0.0022;nan;-1;0.1668;0.0912;0.0267;0.1360 +2020-06-01T12:06:00.0/2020-06-01T12:07:00.0;17.9977;14.0754;12.4908;1.5846;15.3154;35.3563;0.9723;341.0231;17.8313;0.0064;0.0067;0.0008;0.0219;0.0254;0.0087;0.0022;nan;-1;0.1668;0.0912;0.0267;0.1360 +2020-06-01T12:07:00.0/2020-06-01T12:08:00.0;17.9849;14.0637;12.4787;1.5850;15.3114;35.4136;0.9723;341.0233;17.8371;0.0064;0.0066;0.0008;0.0220;0.0255;0.0087;0.0022;nan;-1;0.1668;0.0912;0.0267;0.1360 +2020-06-01T12:08:00.0/2020-06-01T12:09:00.0;17.9719;14.0518;12.4664;1.5854;15.3075;35.4717;0.9723;341.0234;17.8430;0.0064;0.0066;0.0008;0.0220;0.0255;0.0087;0.0022;nan;-1;0.1668;0.0912;0.0267;0.1360 +2020-06-01T12:09:00.0/2020-06-01T12:10:00.0;17.9587;14.0397;12.4540;1.5857;15.3035;35.5308;0.9723;341.0236;17.8488;0.0064;0.0066;0.0008;0.0221;0.0255;0.0087;0.0022;nan;-1;0.1668;0.0912;0.0267;0.1361 +2020-06-01T12:10:00.0/2020-06-01T12:11:00.0;17.9453;14.0274;12.4414;1.5860;15.2994;35.5907;0.9724;341.0238;17.8547;0.0064;0.0066;0.0008;0.0222;0.0256;0.0087;0.0022;nan;-1;0.1668;0.0912;0.0267;0.1361 +2020-06-01T12:11:00.0/2020-06-01T12:12:00.0;17.9316;14.0150;12.4286;1.5864;15.2953;35.6514;0.9724;341.0239;17.8605;0.0064;0.0066;0.0008;0.0223;0.0256;0.0087;0.0022;nan;-1;0.1668;0.0912;0.0267;0.1361 +2020-06-01T12:12:00.0/2020-06-01T12:13:00.0;17.9178;14.0023;12.4157;1.5867;15.2911;35.7130;0.9724;341.0241;17.8664;0.0063;0.0066;0.0008;0.0223;0.0256;0.0087;0.0022;nan;-1;0.1668;0.0912;0.0267;0.1361 +2020-06-01T12:13:00.0/2020-06-01T12:14:00.0;17.9037;13.9895;12.4025;1.5870;15.2869;35.7754;0.9724;341.0243;17.8722;0.0063;0.0066;0.0008;0.0224;0.0257;0.0087;0.0022;nan;-1;0.1668;0.0912;0.0267;0.1361 +2020-06-01T12:14:00.0/2020-06-01T12:15:00.0;17.8895;13.9765;12.3892;1.5873;15.2827;35.8387;0.9724;341.0244;17.8781;0.0063;0.0066;0.0008;0.0225;0.0257;0.0087;0.0022;nan;-1;0.1668;0.0912;0.0267;0.1362 +2020-06-01T12:15:00.0/2020-06-01T12:16:00.0;17.8750;13.9633;12.3757;1.5876;15.2784;35.9028;0.9724;341.0246;17.8839;0.0063;0.0065;0.0008;0.0225;0.0258;0.0087;0.0022;nan;-1;0.1668;0.0912;0.0267;0.1362 +2020-06-01T12:16:00.0/2020-06-01T12:17:00.0;17.8603;13.9499;12.3620;1.5879;15.2741;35.9677;0.9724;341.0247;17.8898;0.0063;0.0065;0.0008;0.0226;0.0258;0.0087;0.0022;nan;-1;0.1668;0.0912;0.0267;0.1362 +2020-06-01T12:17:00.0/2020-06-01T12:18:00.0;17.8454;13.9363;12.3482;1.5882;15.2697;36.0335;0.9724;341.0249;17.8956;0.0063;0.0065;0.0008;0.0227;0.0258;0.0087;0.0022;nan;-1;0.1668;0.0912;0.0267;0.1362 +2020-06-01T12:18:00.0/2020-06-01T12:19:00.0;17.8303;13.9226;12.3342;1.5884;15.2652;36.1000;0.9724;341.0251;17.9015;0.0063;0.0065;0.0008;0.0227;0.0259;0.0087;0.0022;nan;-1;0.1668;0.0912;0.0267;0.1362 +2020-06-01T12:19:00.0/2020-06-01T12:20:00.0;17.8150;13.9087;12.3200;1.5887;15.2608;36.1674;0.9724;341.0252;17.9073;0.0063;0.0065;0.0008;0.0228;0.0259;0.0087;0.0022;nan;-1;0.1668;0.0912;0.0267;0.1363 +2020-06-01T12:20:00.0/2020-06-01T12:21:00.0;17.7995;13.8945;12.3056;1.5890;15.2562;36.2356;0.9724;341.0254;17.9132;0.0062;0.0065;0.0008;0.0229;0.0259;0.0087;0.0022;nan;-1;0.1668;0.0912;0.0267;0.1363 +2020-06-01T12:21:00.0/2020-06-01T12:22:00.0;17.7838;13.8802;12.2910;1.5892;15.2517;36.3046;0.9724;341.0256;17.9190;0.0062;0.0065;0.0008;0.0230;0.0260;0.0087;0.0022;nan;-1;0.1668;0.0912;0.0267;0.1363 +2020-06-01T12:22:00.0/2020-06-01T12:23:00.0;17.7679;13.8657;12.2763;1.5894;15.2470;36.3744;0.9724;341.0257;17.9249;0.0062;0.0064;0.0008;0.0230;0.0260;0.0087;0.0022;nan;-1;0.1668;0.0912;0.0267;0.1363 +2020-06-01T12:23:00.0/2020-06-01T12:24:00.0;17.7517;13.8511;12.2614;1.5897;15.2424;36.4449;0.9724;341.0259;17.9307;0.0062;0.0064;0.0008;0.0231;0.0260;0.0087;0.0022;nan;-1;0.1668;0.0912;0.0267;0.1364 +2020-06-01T12:24:00.0/2020-06-01T12:25:00.0;17.7354;13.8362;12.2463;1.5899;15.2377;36.5163;0.9724;341.0261;17.9366;0.0062;0.0064;0.0008;0.0232;0.0261;0.0087;0.0022;nan;-1;0.1668;0.0912;0.0267;0.1364 +2020-06-01T12:25:00.0/2020-06-01T12:26:00.0;17.7188;13.8212;12.2311;1.5901;15.2329;36.5884;0.9724;341.0262;17.9424;0.0062;0.0064;0.0008;0.0232;0.0261;0.0087;0.0022;nan;-1;0.1668;0.0912;0.0267;0.1364 +2020-06-01T12:26:00.0/2020-06-01T12:27:00.0;17.7021;13.8060;12.2156;1.5903;15.2281;36.6613;0.9724;341.0264;17.9483;0.0062;0.0064;0.0008;0.0233;0.0261;0.0087;0.0022;nan;-1;0.1668;0.0912;0.0267;0.1364 +2020-06-01T12:27:00.0/2020-06-01T12:28:00.0;17.6851;13.7905;12.2000;1.5905;15.2232;36.7350;0.9724;341.0266;17.9541;0.0062;0.0064;0.0008;0.0234;0.0262;0.0087;0.0022;nan;-1;0.1668;0.0912;0.0267;0.1365 +2020-06-01T12:28:00.0/2020-06-01T12:29:00.0;17.6680;13.7750;12.1843;1.5907;15.2183;36.8094;0.9724;341.0267;17.9600;0.0061;0.0064;0.0008;0.0234;0.0262;0.0087;0.0022;nan;-1;0.1668;0.0912;0.0267;0.1365 +2020-06-01T12:29:00.0/2020-06-01T12:30:00.0;17.6506;13.7592;12.1683;1.5909;15.2133;36.8846;0.9724;341.0269;17.9658;0.0061;0.0063;0.0008;0.0235;0.0263;0.0087;0.0022;nan;-1;0.1668;0.0912;0.0267;0.1365 +2020-06-01T12:30:00.0/2020-06-01T12:31:00.0;17.6330;13.7433;12.1522;1.5910;15.2083;36.9606;0.9724;341.0271;17.9717;0.0061;0.0063;0.0008;0.0236;0.0263;0.0087;0.0022;nan;-1;0.1668;0.0912;0.0267;0.1366 +2020-06-01T12:31:00.0/2020-06-01T12:32:00.0;17.6152;13.7271;12.1359;1.5912;15.2033;37.0373;0.9724;341.0272;17.9775;0.0061;0.0063;0.0008;0.0237;0.0263;0.0087;0.0022;nan;-1;0.1668;0.0912;0.0267;0.1366 +2020-06-01T12:32:00.0/2020-06-01T12:33:00.0;17.5972;13.7108;12.1195;1.5914;15.1982;37.1147;0.9724;341.0274;17.9834;0.0061;0.0063;0.0008;0.0237;0.0264;0.0087;0.0022;nan;-1;0.1668;0.0912;0.0267;0.1366 +2020-06-01T12:33:00.0/2020-06-01T12:34:00.0;17.5790;13.6943;12.1028;1.5915;15.1930;37.1929;0.9724;341.0276;17.9892;0.0061;0.0063;0.0008;0.0238;0.0264;0.0087;0.0022;nan;-1;0.1668;0.0912;0.0267;0.1366 +2020-06-01T12:34:00.0/2020-06-01T12:35:00.0;17.5606;13.6777;12.0860;1.5917;15.1878;37.2718;0.9724;341.0277;17.9951;0.0061;0.0063;0.0008;0.0239;0.0264;0.0087;0.0022;nan;-1;0.1668;0.0912;0.0267;0.1367 +2020-06-01T12:35:00.0/2020-06-01T12:36:00.0;17.5420;13.6608;12.0691;1.5918;15.1826;37.3515;0.9724;341.0279;18.0010;0.0061;0.0063;0.0007;0.0239;0.0265;0.0087;0.0022;nan;-1;0.1668;0.0912;0.0267;0.1367 +2020-06-01T12:36:00.0/2020-06-01T12:37:00.0;17.5233;13.6438;12.0519;1.5919;15.1773;37.4318;0.9724;341.0280;18.0068;0.0060;0.0063;0.0007;0.0240;0.0265;0.0087;0.0022;nan;-1;0.1668;0.0912;0.0267;0.1367 +2020-06-01T12:37:00.0/2020-06-01T12:38:00.0;17.5042;13.6266;12.0346;1.5920;15.1719;37.5129;0.9724;341.0282;18.0127;0.0060;0.0062;0.0007;0.0241;0.0265;0.0087;0.0022;nan;-1;0.1668;0.0912;0.0267;0.1368 +2020-06-01T12:38:00.0/2020-06-01T12:39:00.0;17.4850;13.6092;12.0171;1.5921;15.1665;37.5947;0.9724;341.0284;18.0185;0.0060;0.0062;0.0007;0.0242;0.0266;0.0087;0.0022;nan;-1;0.1668;0.0912;0.0267;0.1368 +2020-06-01T12:39:00.0/2020-06-01T12:40:00.0;17.4656;13.5917;11.9995;1.5922;15.1611;37.6772;0.9724;341.0285;18.0244;0.0060;0.0062;0.0007;0.0242;0.0266;0.0087;0.0022;nan;-1;0.1668;0.0912;0.0267;0.1368 +2020-06-01T12:40:00.0/2020-06-01T12:41:00.0;17.4460;13.5739;11.9816;1.5923;15.1555;37.7604;0.9724;341.0287;18.0302;0.0060;0.0062;0.0007;0.0243;0.0266;0.0087;0.0022;nan;-1;0.1668;0.0912;0.0267;0.1369 +2020-06-01T12:41:00.0/2020-06-01T12:42:00.0;17.4262;13.5560;11.9637;1.5924;15.1500;37.8443;0.9724;341.0289;18.0361;0.0060;0.0062;0.0007;0.0244;0.0267;0.0087;0.0022;nan;-1;0.1668;0.0912;0.0267;0.1369 +2020-06-01T12:42:00.0/2020-06-01T12:43:00.0;17.4062;13.5379;11.9455;1.5924;15.1444;37.9289;0.9724;341.0290;18.0419;0.0060;0.0062;0.0007;0.0244;0.0267;0.0087;0.0022;nan;-1;0.1668;0.0912;0.0267;0.1369 +2020-06-01T12:43:00.0/2020-06-01T12:44:00.0;17.3860;13.5197;11.9272;1.5925;15.1387;38.0141;0.9724;341.0292;18.0478;0.0060;0.0062;0.0007;0.0245;0.0267;0.0087;0.0022;nan;-1;0.1668;0.0912;0.0267;0.1370 +2020-06-01T12:44:00.0/2020-06-01T12:45:00.0;17.3656;13.5012;11.9087;1.5925;15.1330;38.1001;0.9724;341.0294;18.0536;0.0059;0.0061;0.0007;0.0246;0.0268;0.0087;0.0022;nan;-1;0.1668;0.0912;0.0267;0.1370 +2020-06-01T12:45:00.0/2020-06-01T12:46:00.0;17.3450;13.4826;11.8900;1.5926;15.1273;38.1867;0.9724;341.0295;18.0595;0.0059;0.0061;0.0007;0.0246;0.0268;0.0087;0.0022;nan;-1;0.1668;0.0912;0.0267;0.1370 +2020-06-01T12:46:00.0/2020-06-01T12:47:00.0;17.3242;13.4638;11.8712;1.5926;15.1215;38.2740;0.9724;341.0297;18.0653;0.0059;0.0061;0.0007;0.0247;0.0269;0.0087;0.0022;nan;-1;0.1668;0.0912;0.0267;0.1371 +2020-06-01T12:47:00.0/2020-06-01T12:48:00.0;17.3032;13.4449;11.8522;1.5926;15.1156;38.3620;0.9724;341.0299;18.0712;0.0059;0.0061;0.0007;0.0248;0.0269;0.0087;0.0022;nan;-1;0.1668;0.0912;0.0267;0.1371 +2020-06-01T12:48:00.0/2020-06-01T12:49:00.0;17.2820;13.4257;11.8331;1.5927;15.1097;38.4506;0.9724;341.0300;18.0770;0.0059;0.0061;0.0007;0.0249;0.0269;0.0087;0.0022;nan;-1;0.1668;0.0912;0.0267;0.1371 +2020-06-01T12:49:00.0/2020-06-01T12:50:00.0;17.2605;13.4064;11.8137;1.5927;15.1037;38.5398;0.9724;341.0302;18.0829;0.0059;0.0061;0.0007;0.0249;0.0270;0.0087;0.0022;nan;-1;0.1668;0.0912;0.0267;0.1372 +2020-06-01T12:50:00.0/2020-06-01T12:51:00.0;17.2390;13.3869;11.7943;1.5927;15.0977;38.6297;0.9724;341.0304;18.0887;0.0059;0.0061;0.0007;0.0250;0.0270;0.0087;0.0022;nan;-1;0.1668;0.0912;0.0267;0.1372 +2020-06-01T12:51:00.0/2020-06-01T12:52:00.0;17.2171;13.3673;11.7746;1.5926;15.0916;38.7203;0.9724;341.0305;18.0946;0.0059;0.0060;0.0007;0.0251;0.0270;0.0087;0.0022;nan;-1;0.1668;0.0912;0.0267;0.1372 +2020-06-01T12:52:00.0/2020-06-01T12:53:00.0;17.1952;13.3474;11.7548;1.5926;15.0855;38.8115;0.9724;341.0307;18.1004;0.0058;0.0060;0.0007;0.0251;0.0271;0.0087;0.0022;nan;-1;0.1668;0.0912;0.0267;0.1373 +2020-06-01T12:53:00.0/2020-06-01T12:54:00.0;17.1730;13.3274;11.7348;1.5926;15.0793;38.9033;0.9724;341.0308;18.1063;0.0058;0.0060;0.0007;0.0252;0.0271;0.0087;0.0022;nan;-1;0.1668;0.0912;0.0267;0.1373 +2020-06-01T12:54:00.0/2020-06-01T12:55:00.0;17.1506;13.3072;11.7147;1.5926;15.0731;38.9957;0.9724;341.0310;18.1121;0.0058;0.0060;0.0007;0.0253;0.0271;0.0087;0.0022;nan;-1;0.1668;0.0912;0.0267;0.1374 +2020-06-01T12:55:00.0/2020-06-01T12:56:00.0;17.1280;13.2869;11.6944;1.5925;15.0668;39.0888;0.9724;341.0312;18.1180;0.0058;0.0060;0.0007;0.0253;0.0272;0.0087;0.0022;nan;-1;0.1668;0.0912;0.0267;0.1374 +2020-06-01T12:56:00.0/2020-06-01T12:57:00.0;17.1052;13.2664;11.6739;1.5924;15.0605;39.1825;0.9724;341.0313;18.1238;0.0058;0.0060;0.0007;0.0254;0.0272;0.0086;0.0022;nan;-1;0.1668;0.0912;0.0267;0.1374 +2020-06-01T12:57:00.0/2020-06-01T12:58:00.0;17.0822;13.2457;11.6533;1.5924;15.0541;39.2768;0.9724;341.0315;18.1297;0.0058;0.0060;0.0007;0.0255;0.0272;0.0086;0.0022;nan;-1;0.1668;0.0912;0.0267;0.1375 +2020-06-01T12:58:00.0/2020-06-01T12:59:00.0;17.0591;13.2248;11.6325;1.5923;15.0476;39.3717;0.9724;341.0317;18.1355;0.0058;0.0060;0.0007;0.0256;0.0273;0.0086;0.0022;nan;-1;0.1668;0.0912;0.0267;0.1375 +2020-06-01T12:59:00.0/2020-06-01T13:00:00.0;17.0357;13.2038;11.6116;1.5922;15.0411;39.4672;0.9724;341.0318;18.1414;0.0058;0.0059;0.0007;0.0256;0.0273;0.0086;0.0022;nan;-1;0.1668;0.0912;0.0267;0.1376 From 673bf3baa44fb1fc844ad232b5de927273904cb0 Mon Sep 17 00:00:00 2001 From: AdamRJensen Date: Fri, 26 Feb 2021 16:57:44 +0100 Subject: [PATCH 13/57] Convert to get_cams to add support for CAMS Radiation --- pvlib/iotools/cams.py | 186 +++++++++++++++++++++++++++--------------- 1 file changed, 118 insertions(+), 68 deletions(-) diff --git a/pvlib/iotools/cams.py b/pvlib/iotools/cams.py index ce9bc352ef..df6840b24d 100644 --- a/pvlib/iotools/cams.py +++ b/pvlib/iotools/cams.py @@ -8,28 +8,29 @@ import io -MCCLEAR_COLUMNS = ['Observation period', 'TOA', 'Clear sky GHI', - 'Clear sky BHI', 'Clear sky DHI', 'Clear sky BNI'] +CAMS_INTEGRATED_COLUMNS = ['TOA', 'Clear sky GHI', 'Clear sky BHI', + 'Clear sky DHI', 'Clear sky BNI', + 'GHI', 'BHI', 'DHI', 'BNI', 'GHI no corr', + 'BHI no corr', 'DHI no corr', 'BNI no corr'] -MCCLEAR_VERBOSE_COLUMNS = ['sza', 'summer/winter split', 'tco3', 'tcwv', - 'AOD BC', 'AOD DU', 'AOD SS', 'AOD OR', 'AOD SU', - 'AOD NI', 'AOD AM', 'alpha', 'Aerosol type', - 'fiso', 'fvol', 'fgeo', 'albedo'] - -# Dictionary mapping CAMS MCCLEAR variables to pvlib names -MCCLEAR_VARIABLE_MAP = { +# Dictionary mapping CAMS McClear and Radiation variables to pvlib names +CAMS_VARIABLE_MAP = { 'TOA': 'ghi_extra', 'Clear sky GHI': 'ghi_clear', 'Clear sky BHI': 'bhi_clear', 'Clear sky DHI': 'dhi_clear', 'Clear sky BNI': 'dni_clear', + 'GHI': 'ghi', + 'BHI': 'bhi', + 'DHI': 'dhi', + 'BNI': 'dni', 'sza': 'solar_zenith', } # Dictionary mapping Python time steps to CAMS time step format -TIME_STEPS_MAP = {'1min': 'PT01M', '15min': 'PT15M', '1h': 'PT01H', '1d': 'P01D', - '1M': 'P01M'} +TIME_STEPS_MAP = {'1min': 'PT01M', '15min': 'PT15M', '1h': 'PT01H', + '1d': 'P01D', '1M': 'P01M'} TIME_STEPS_IN_HOURS = {'1min': 1/60, '15min': 15/60, '1h': 1, '1d': 24} @@ -40,16 +41,16 @@ '0 year 1 month 0 day 0 h 0 min 0 s': '1M'} -def get_cams_mcclear(start_date, end_date, latitude, longitude, email, - altitude=None, time_step='1h', time_ref='UT', - integrated=False, label=None, verbose=False, - map_variables=True, server='www.soda-is.com'): +def get_cams(start_date, end_date, latitude, longitude, email, + service='mcclear', altitude=None, time_step='1h', time_ref='UT', + verbose=False, integrated=False, label=None, map_variables=True, + server='www.soda-is.com'): """ - Retrieve time-series of clear-sky global, beam, and diffuse radiation - anywhere in the world from CAMS McClear [1]_ using the WGET service [2]_. + Retrieve time-series of radiation and/or clear-sky global, beam, and + diffuse radiation CAMS [2]_ using the WGET service [3]_. - Geographical coverage: wordwide + Geographical coverage: -66 to 66 latitude/longitude (radiation) and wordwide (McClear) Time coverage: 2004-01-01 to two days ago Access: free, but requires registration, see [1]_ Requests: max. 100 per day @@ -70,26 +71,34 @@ def get_cams_mcclear(start_date, end_date, latitude, longitude, email, NASA SRTM database email: str Email address linked to a SoDa account + service: {'mcclear', 'cams_radiation'} + Specify which whether to retrieve CAMS Radiation or McClear parameters time_step: str, {'1min', '15min', '1h', '1d', '1M'}, default: '1h' Time step of the time series, either 1 minute, 15 minute, hourly, daily, or monthly. time_reference: str, {'UT', 'TST'}, default: 'UT' 'UT' (universal time) or 'TST' (True Solar Time) + verbose: boolean, default: False + Verbose mode outputs additional parameters (aerosols). Only avaiable + for 1 minute and universal time. See [1] for parameter description. integrated: boolean, default False Whether to return integrated irradiation values (Wh/m^2) from CAMS or average irradiance values (W/m^2) as is more commonly used label: {‘right’, ‘left’}, default: None Which bin edge label to label bucket with. The default is ‘left’ for all frequency offsets except for ‘M’ which has a default of ‘right’. - verbose: boolean, default: False - Verbose mode outputs additional parameters (aerosols). Only avaiable - for 1 minute and universal time. See [1] for parameter description. map_variables: bool, default: True When true, renames columns of the Dataframe to pvlib variable names - where applicable. See variable MCCLEAR_VARIABLE_MAP. + where applicable. See variable CAMS_VARIABLE_MAP. server: str, default: 'www.soda-is.com' Main server (www.soda-is.com) or backup mirror server (pro.soda-is.com) + Returns + ------- + data: pandas.DataFrame + timeseries data, see Notes for columns + meta: dict + metadata for the requested time-series Notes ---------- @@ -103,30 +112,41 @@ def get_cams_mcclear(start_date, end_date, latitude, longitude, email, Observation period str Beginning/end of time period TOA, ghi_extra float Horizontal radiation at top of atmosphere Clear sky GHI, ghi_clear float Clear sky global radiation on horizontal - Clear sky BHI, bhi_clear float Clear sky beam radiation on horizontal + Clear sky BHI, bhi float Clear sky beam radiation on horizontal Clear sky DHI, dhi_clear float Clear sky diffuse radiation on horizontal Clear sky BNI, dni_clear float Clear sky beam radiation normal to sun + GHI, ghi* float Global horizontal radiation + BHI, bhi* float Beam (direct) radiation on horizontal + DHI, dhi* float Diffuse horizontal radiation + BNI, dni* float Beam (direct) radiation normal to the sun + Reliability* float Fraction of reliable data in summarization ======================= ====== ========================================== - For the returned units see the integrated argument. For description of - additional output parameters in verbose mode, see [1]. + *Parameters only returned if service='cams_radiation'. For description of + additional output parameters in verbose mode, see [1]_ and [2]_. + + The returned units for the radiation parameters depends on the integrated + argument, i.e. integrated=False returns units of W/m2, whereas + integrated=True returns units of Wh/m2. Note that it is recommended to specify the latitude and longitude to at least the fourth decimal place. Variables corresponding to standard pvlib variables are renamed, e.g. `sza` becomes `solar_zenith`. See the - `pvlib.iotools.cams.MCCLEAR_VARIABLE_MAP` dict for the complete mapping. + `pvlib.iotools.cams.CAMS_VARIABLE_MAP` dict for the complete mapping. See Also -------- - pvlib.iotools.read_cams_mcclear, pvlib.iotools.parse_cams_mcclear + pvlib.iotools.read_cams, pvlib.iotools.parse_cams References ---------- - .. [1] `CAMS McClear Service Info + .. [1] `CAMS Radiation Service Info + `_ + .. [2] `CAMS McClear Service Info `_ - .. [2] `CAMS McClear Automatic Access + .. [3] `CAMS McClear Automatic Access `_ """ @@ -134,15 +154,14 @@ def get_cams_mcclear(start_date, end_date, latitude, longitude, email, time_step_str = TIME_STEPS_MAP[time_step] else: print('WARNING: time step not recognized, 1 hour time step used!') - time_step_str = 'PT01H' + time_step, time_step_str = '1h', 'PT01H' - names = MCCLEAR_COLUMNS - if verbose: - if (time_step == '1min') & (time_ref == 'UT'): - names += MCCLEAR_VERBOSE_COLUMNS - else: - verbose = False - print("Verbose mode only supports 1 min. UT time series!") + if (verbose is True) & ((time_step != '1min') | (time_ref != 'UT')): + verbose = False + print("Verbose mode only supports 1 min. UT time series!") + + # Format verbose variable to the required format: {'true', 'false'} + verbose = str(verbose).lower() if altitude is None: # Let SoDa get elevation from the NASA SRTM database altitude = -999 @@ -152,52 +171,72 @@ def get_cams_mcclear(start_date, end_date, latitude, longitude, email, end_date = end_date.strftime('%Y-%m-%d') email = email.replace('@', '%2540') # Format email address - - # Format verbose variable to the required format: {'true', 'false'} - verbose = str(verbose).lower() + service = 'get_{}'.format(service) # Format CAMS service string # Manual format the request url, due to uncommon usage of & and ; in url url = ("http://{}/service/wps?Service=WPS&Request=Execute&" - "Identifier=get_mcclear&version=1.0.0&RawDataOutput=irradiation&" + "Identifier={}&version=1.0.0&RawDataOutput=irradiation&" "DataInputs=latitude={};longitude={};altitude={};" "date_begin={};date_end={};time_ref={};summarization={};" "username={};verbose={}" - ).format(server, latitude, longitude, altitude, start_date, + ).format(server, service, latitude, longitude, altitude, start_date, end_date, time_ref, time_step_str, email, verbose) res = requests.get(url) # Invalid requests returns helpful XML error message if res.headers['Content-Type'] == 'application/xml': - print('REQUEST ERROR MESSAGE:') - print(res.text.split('ows:ExceptionText')[1][1:-2]) - + errors = res.text.split('ows:ExceptionText')[1][1:-2] + raise requests.HTTPError(errors, response=res) # Check if returned file is a csv data file elif res.headers['Content-Type'] == 'application/csv': fbuf = io.StringIO(res.content.decode('utf-8')) - data, meta = parse_cams_mcclear(fbuf, integrated=integrated, - label=label, - map_variables=map_variables) + data, meta = parse_cams(fbuf, integrated=integrated, label=label, + map_variables=map_variables) return data, meta else: - print('Error in recognizing the file content occurred!') + print('Error! File content type not recognized.') -def parse_cams_mcclear(fbuf, integrated=False, label=None, map_variables=True): +def parse_cams(fbuf, integrated=False, label=None, map_variables=True): """ - Parse a CAMS McClear file. CAMS McClear is described in [1]_. + Parse a CAMS Radiation or McClear file. The CAMS Radiation and McClear + services are described in [1]_ and [2]_. + + Parameters + ---------- + fbuf: file-like object + File-like object containing data to read. + integrated: boolean, default False + Whether to return integrated irradiation values (Wh/m^2) from CAMS or + average irradiance values (W/m^2) as is more commonly used + label: {‘right’, ‘left’}, default: None + Which bin edge label to label bucket with. The default is ‘left’ for + all frequency offsets except for ‘M’ which has a default of ‘right’. + map_variables: bool, default: True + When true, renames columns of the Dataframe to pvlib variable names + where applicable. See variable CAMS_VARIABLE_MAP. + + Returns + ------- + data: pandas.DataFrame + timeseries data from CAMS Radiation or McClear + meta: dict + metadata of the CAMS Radiation or McClear time-series See Also -------- - pvlib.iotools.read_cams_mcclear, pvlib.iotools.get_cams_mcclear + pvlib.iotools.read_cams, pvlib.iotools.get_cams References ---------- - .. [1] `CAMS McClear Service Info + .. [1] `CAMS Radiation Service Info + `_ + .. [2] `CAMS McClear Service Info `_ """ meta = {} - # Initial lines of the file contain meta-data, which all start with # + # Initial lines starting with # contain meta-data while True: line = fbuf.readline().rstrip('\n') if line.startswith('# Observation period'): @@ -213,7 +252,7 @@ def parse_cams_mcclear(fbuf, integrated=False, label=None, map_variables=True): meta['Longitude (positive East, ISO 19115)'] = \ float(meta['Longitude (positive East, ISO 19115)']) meta['Altitude (m)'] = float(meta['Altitude (m)']) - meta['Clear sky radiation unit'] = {True:'Wh/m2', False:'W/m2'}[integrated] + meta['Unit for radiation'] = {True:'Wh/m2', False:'W/m2'}[integrated] # Determine the time_step from the meta-data dictionary time_step = SUMMATION_PERIOD_TO_TIME_STEP[ @@ -242,8 +281,8 @@ def parse_cams_mcclear(fbuf, integrated=False, label=None, map_variables=True): if (time_step == '1M') & (label != 'left'): data.index = data.index - pd.Timedelta(days=1) - if not integrated: # Convert from Wh/m2 to W/m2 - integrated_cols = MCCLEAR_COLUMNS[1:6] + if not integrated: # Convert radiation values from Wh/m2 to W/m2 + integrated_cols = [c for c in CAMS_INTEGRATED_COLUMNS if c in data.columns] if time_step == '1M': time_delta = (pd.to_datetime(obs_period.str[1]) @@ -256,37 +295,48 @@ def parse_cams_mcclear(fbuf, integrated=False, label=None, map_variables=True): TIME_STEPS_IN_HOURS[time_step]) if map_variables: - data = data.rename(columns=MCCLEAR_VARIABLE_MAP) + data = data.rename(columns=CAMS_VARIABLE_MAP) return data, meta -def read_cams_mcclear(filename): +def read_cams(filename, integrated=False, label=None, map_variables=True): """ - Read a CAMS McClear file. CAMS McClear is described in [1]_. + Read a CAMS Radiation or McClear file. CAMS radiation and McClear is + described in [1]_ and [2]_, respectively. Parameters ---------- filename: str Filename of a file containing data to read. + integrated: boolean, default False + Whether to return integrated irradiation values (Wh/m^2) from CAMS or + average irradiance values (W/m^2) as is more commonly used + label: {‘right’, ‘left’}, default: None + Which bin edge label to label bucket with. The default is ‘left’ for + all frequency offsets except for ‘M’ which has a default of ‘right’. + map_variables: bool, default: True + When true, renames columns of the Dataframe to pvlib variable names + where applicable. See variable CAMS_VARIABLE_MAP. Returns ------- - data : pandas.DataFrame - timeseries data from CAMS McClear - meta : dict - metadata from CAMS McClear, see - :func:`pvlib.iotools.parse_cams_mcclear` for fields + data: pandas.DataFrame + timeseries data from CAMS Radiation or McClear + meta: dict + metadata from CAMS Radiation or McClear See Also -------- - pvlib.iotools.parse_cams_mcclear, pvlib.iotools.get_cams_mcclear + pvlib.iotools.parse_cams, pvlib.iotools.get_cams References ---------- - .. [1] `CAMS McClear Service Info + .. [1] `CAMS Radiation Service Info + `_ + .. [2] `CAMS McClear Service Info `_ """ with open(str(filename), 'r') as fbuf: - content = parse_cams_mcclear(fbuf) + content = parse_cams(fbuf) return content From c6764d3e909e1d1ad3e09c8b2cf6c38e7385a685 Mon Sep 17 00:00:00 2001 From: AdamRJensen Date: Fri, 26 Feb 2021 17:01:21 +0100 Subject: [PATCH 14/57] Fixed stickler issues --- pvlib/iotools/cams.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/pvlib/iotools/cams.py b/pvlib/iotools/cams.py index df6840b24d..cd5bf90e35 100644 --- a/pvlib/iotools/cams.py +++ b/pvlib/iotools/cams.py @@ -35,7 +35,7 @@ TIME_STEPS_IN_HOURS = {'1min': 1/60, '15min': 15/60, '1h': 1, '1d': 24} SUMMATION_PERIOD_TO_TIME_STEP = {'0 year 0 month 0 day 0 h 1 min 0 s': '1min', - '0 year 0 month 0 day 0 h 15 min 0 s': '15min', + '0 year 0 month 0 day 0 h 15 min 0 s': '15min', # noqa '0 year 0 month 0 day 1 h 0 min 0 s': '1h', '0 year 0 month 1 day 0 h 0 min 0 s': '1d', '0 year 1 month 0 day 0 h 0 min 0 s': '1M'} @@ -50,7 +50,8 @@ def get_cams(start_date, end_date, latitude, longitude, email, diffuse radiation CAMS [2]_ using the WGET service [3]_. - Geographical coverage: -66 to 66 latitude/longitude (radiation) and wordwide (McClear) + Geographical coverage: Wordwide for CAMS McClear and -66 to 66 for both + latitude and longitude for CAMS Radiation Time coverage: 2004-01-01 to two days ago Access: free, but requires registration, see [1]_ Requests: max. 100 per day @@ -127,7 +128,7 @@ def get_cams(start_date, end_date, latitude, longitude, email, The returned units for the radiation parameters depends on the integrated argument, i.e. integrated=False returns units of W/m2, whereas - integrated=True returns units of Wh/m2. + integrated=True returns units of Wh/m2. Note that it is recommended to specify the latitude and longitude to at least the fourth decimal place. @@ -252,7 +253,7 @@ def parse_cams(fbuf, integrated=False, label=None, map_variables=True): meta['Longitude (positive East, ISO 19115)'] = \ float(meta['Longitude (positive East, ISO 19115)']) meta['Altitude (m)'] = float(meta['Altitude (m)']) - meta['Unit for radiation'] = {True:'Wh/m2', False:'W/m2'}[integrated] + meta['Unit for radiation'] = {True: 'Wh/m2', False: 'W/m2'}[integrated] # Determine the time_step from the meta-data dictionary time_step = SUMMATION_PERIOD_TO_TIME_STEP[ @@ -282,7 +283,7 @@ def parse_cams(fbuf, integrated=False, label=None, map_variables=True): data.index = data.index - pd.Timedelta(days=1) if not integrated: # Convert radiation values from Wh/m2 to W/m2 - integrated_cols = [c for c in CAMS_INTEGRATED_COLUMNS if c in data.columns] + integrated_cols = [c for c in CAMS_INTEGRATED_COLUMNS if c in data.columns] # noqa if time_step == '1M': time_delta = (pd.to_datetime(obs_period.str[1]) From 225b0e137366a17473a9c6d697cd63a6cda8acfe Mon Sep 17 00:00:00 2001 From: AdamRJensen Date: Fri, 26 Feb 2021 17:08:01 +0100 Subject: [PATCH 15/57] Update function names to just 'cams' --- docs/sphinx/source/api.rst | 4 +++- docs/sphinx/source/whatsnew/v0.9.0.rst | 4 ++-- pvlib/iotools/__init__.py | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/docs/sphinx/source/api.rst b/docs/sphinx/source/api.rst index 31204b6f0d..acd0a6856f 100644 --- a/docs/sphinx/source/api.rst +++ b/docs/sphinx/source/api.rst @@ -484,7 +484,9 @@ relevant to solar energy modeling. iotools.get_pvgis_tmy iotools.read_pvgis_tmy iotools.read_bsrn - iotools.get_cams_mcclear + iotools.get_cams + iotools.read_cams + iotools.parse_cams A :py:class:`~pvlib.location.Location` object may be created from metadata in some files. diff --git a/docs/sphinx/source/whatsnew/v0.9.0.rst b/docs/sphinx/source/whatsnew/v0.9.0.rst index 497499aa75..fd941bbcfb 100644 --- a/docs/sphinx/source/whatsnew/v0.9.0.rst +++ b/docs/sphinx/source/whatsnew/v0.9.0.rst @@ -64,8 +64,8 @@ Enhancements ~~~~~~~~~~~~ * Add :func:`~pvlib.iotools.read_bsrn` for reading BSRN solar radiation data files. (:pull:`1145`, :issue:`1015`) -* Add :func:`~pvlib.iotools.get_cams_mcclear` for retrieving CAMS McClear - clear-sky radiation time series +* Add :func:`~pvlib.iotools.get_cams` and :func:`~pvlib.iotools.read_cams` + for retrieving and reading CAMS Radiation and McClear radiation time series files. (:pull:`1175`) * In :py:class:`~pvlib.modelchain.ModelChain`, attributes which contain output of models are now collected into ``ModelChain.results``. diff --git a/pvlib/iotools/__init__.py b/pvlib/iotools/__init__.py index 892c18edac..3288924a3a 100644 --- a/pvlib/iotools/__init__.py +++ b/pvlib/iotools/__init__.py @@ -14,4 +14,4 @@ from pvlib.iotools.psm3 import parse_psm3 # noqa: F401 from pvlib.iotools.pvgis import get_pvgis_tmy, read_pvgis_tmy # noqa: F401 from pvlib.iotools.bsrn import read_bsrn # noqa: F401 -from pvlib.iotools.cams import get_cams_mcclear # noqa: F401 +from pvlib.iotools.cams import get_cams, read_cams, parse_cams # noqa: F401 From 924c58aa38b88fd69c88bcc849590c4c7abd0bff Mon Sep 17 00:00:00 2001 From: AdamRJensen Date: Sat, 27 Feb 2021 15:26:11 +0100 Subject: [PATCH 16/57] Update return description --- pvlib/iotools/cams.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/pvlib/iotools/cams.py b/pvlib/iotools/cams.py index cd5bf90e35..5e155a91a6 100644 --- a/pvlib/iotools/cams.py +++ b/pvlib/iotools/cams.py @@ -97,12 +97,15 @@ def get_cams(start_date, end_date, latitude, longitude, email, Returns ------- data: pandas.DataFrame - timeseries data, see Notes for columns + Timeseries data, see Notes for columns meta: dict - metadata for the requested time-series + Metadata for the requested time-series Notes ---------- + In order to use the CAMS services, users must registre for a free SoDa + account using an email addres [1]_. + The returned data Dataframe includes the following fields: ======================= ====== ========================================== @@ -221,9 +224,9 @@ def parse_cams(fbuf, integrated=False, label=None, map_variables=True): Returns ------- data: pandas.DataFrame - timeseries data from CAMS Radiation or McClear + Timeseries data from CAMS Radiation or McClear meta: dict - metadata of the CAMS Radiation or McClear time-series + Metadata avaiable in the file. See Also -------- @@ -323,9 +326,9 @@ def read_cams(filename, integrated=False, label=None, map_variables=True): Returns ------- data: pandas.DataFrame - timeseries data from CAMS Radiation or McClear + Timeseries data from CAMS Radiation or McClear meta: dict - metadata from CAMS Radiation or McClear + Metadata avaiable in the file. See Also -------- From cbc116dd80773a20061cfac04da54b7f10ff7b8c Mon Sep 17 00:00:00 2001 From: AdamRJensen Date: Sat, 27 Feb 2021 16:35:11 +0100 Subject: [PATCH 17/57] Rename to cams_radiation And minor updates to the documentation --- docs/sphinx/source/api.rst | 6 +-- docs/sphinx/source/whatsnew/v0.9.0.rst | 5 ++- pvlib/iotools/__init__.py | 2 +- pvlib/iotools/cams.py | 51 +++++++++++++++----------- 4 files changed, 36 insertions(+), 28 deletions(-) diff --git a/docs/sphinx/source/api.rst b/docs/sphinx/source/api.rst index acd0a6856f..49ded6f1fa 100644 --- a/docs/sphinx/source/api.rst +++ b/docs/sphinx/source/api.rst @@ -484,9 +484,9 @@ relevant to solar energy modeling. iotools.get_pvgis_tmy iotools.read_pvgis_tmy iotools.read_bsrn - iotools.get_cams - iotools.read_cams - iotools.parse_cams + iotools.get_cams_radiation + iotools.read_cams_radiation + iotools.parse_cams_radiation A :py:class:`~pvlib.location.Location` object may be created from metadata in some files. diff --git a/docs/sphinx/source/whatsnew/v0.9.0.rst b/docs/sphinx/source/whatsnew/v0.9.0.rst index fd941bbcfb..5038a36ff2 100644 --- a/docs/sphinx/source/whatsnew/v0.9.0.rst +++ b/docs/sphinx/source/whatsnew/v0.9.0.rst @@ -64,8 +64,9 @@ Enhancements ~~~~~~~~~~~~ * Add :func:`~pvlib.iotools.read_bsrn` for reading BSRN solar radiation data files. (:pull:`1145`, :issue:`1015`) -* Add :func:`~pvlib.iotools.get_cams` and :func:`~pvlib.iotools.read_cams` - for retrieving and reading CAMS Radiation and McClear radiation time series +* Add :func:`~pvlib.iotools.get_cams_radiatoin` and + :func:`~pvlib.iotools.read_cams_radiation` + for retrieving and reading CAMS Radiation and McClear time-series files. (:pull:`1175`) * In :py:class:`~pvlib.modelchain.ModelChain`, attributes which contain output of models are now collected into ``ModelChain.results``. diff --git a/pvlib/iotools/__init__.py b/pvlib/iotools/__init__.py index 3288924a3a..65137d9be1 100644 --- a/pvlib/iotools/__init__.py +++ b/pvlib/iotools/__init__.py @@ -14,4 +14,4 @@ from pvlib.iotools.psm3 import parse_psm3 # noqa: F401 from pvlib.iotools.pvgis import get_pvgis_tmy, read_pvgis_tmy # noqa: F401 from pvlib.iotools.bsrn import read_bsrn # noqa: F401 -from pvlib.iotools.cams import get_cams, read_cams, parse_cams # noqa: F401 +from pvlib.iotools.cams import get_cams_radiation, read_cams_radiation, parse_cams_radiation # noqa: F401 diff --git a/pvlib/iotools/cams.py b/pvlib/iotools/cams.py index 5e155a91a6..fc45989bf3 100644 --- a/pvlib/iotools/cams.py +++ b/pvlib/iotools/cams.py @@ -8,13 +8,13 @@ import io -CAMS_INTEGRATED_COLUMNS = ['TOA', 'Clear sky GHI', 'Clear sky BHI', - 'Clear sky DHI', 'Clear sky BNI', - 'GHI', 'BHI', 'DHI', 'BNI', 'GHI no corr', - 'BHI no corr', 'DHI no corr', 'BNI no corr'] +CAMS_RADIATION_INTEGRATED_COLUMNS = [ + 'TOA', 'Clear sky GHI', 'Clear sky BHI', 'Clear sky DHI', 'Clear sky BNI', + 'GHI', 'BHI', 'DHI', 'BNI', + 'GHI no corr', 'BHI no corr', 'DHI no corr', 'BNI no corr'] # Dictionary mapping CAMS McClear and Radiation variables to pvlib names -CAMS_VARIABLE_MAP = { +CAMS_RADIATION_VARIABLE_MAP = { 'TOA': 'ghi_extra', 'Clear sky GHI': 'ghi_clear', 'Clear sky BHI': 'bhi_clear', @@ -41,20 +41,20 @@ '0 year 1 month 0 day 0 h 0 min 0 s': '1M'} -def get_cams(start_date, end_date, latitude, longitude, email, - service='mcclear', altitude=None, time_step='1h', time_ref='UT', - verbose=False, integrated=False, label=None, map_variables=True, - server='www.soda-is.com'): +def get_cams_radiation(start_date, end_date, latitude, longitude, email, + service='mcclear', altitude=None, time_step='1h', + time_ref='UT', verbose=False, integrated=False, + label=None, map_variables=True, + server='www.soda-is.com'): """ Retrieve time-series of radiation and/or clear-sky global, beam, and - diffuse radiation CAMS [2]_ using the WGET service [3]_. + diffuse radiation from CAMS [2]_ using the WGET service [3]_. - - Geographical coverage: Wordwide for CAMS McClear and -66 to 66 for both - latitude and longitude for CAMS Radiation Time coverage: 2004-01-01 to two days ago Access: free, but requires registration, see [1]_ Requests: max. 100 per day + Geographical coverage: Wordwide for CAMS McClear and -66° to 66° in both + latitude and longitude for CAMS Radiation Parameters @@ -102,11 +102,11 @@ def get_cams(start_date, end_date, latitude, longitude, email, Metadata for the requested time-series Notes - ---------- + ----- In order to use the CAMS services, users must registre for a free SoDa account using an email addres [1]_. - The returned data Dataframe includes the following fields: + The returned data DataFrame includes the following fields: ======================= ====== ========================================== Key, mapped key Format Description @@ -144,6 +144,12 @@ def get_cams(start_date, end_date, latitude, longitude, email, -------- pvlib.iotools.read_cams, pvlib.iotools.parse_cams + Raises + ------ + requests.HTTPError + If the request is invalid, then an XML file is returned by the CAMS + service and the error message will be raised as an expcetion. + References ---------- .. [1] `CAMS Radiation Service Info @@ -153,7 +159,6 @@ def get_cams(start_date, end_date, latitude, longitude, email, .. [3] `CAMS McClear Automatic Access `_ """ - if time_step in TIME_STEPS_MAP.keys(): time_step_str = TIME_STEPS_MAP[time_step] else: @@ -204,8 +209,8 @@ def get_cams(start_date, end_date, latitude, longitude, email, def parse_cams(fbuf, integrated=False, label=None, map_variables=True): """ - Parse a CAMS Radiation or McClear file. The CAMS Radiation and McClear - services are described in [1]_ and [2]_. + Parse a file-like buffer with data in the format of a CAMS Radiation or + McClear file. The CAMS servicess are described in [1]_ and [2]_. Parameters ---------- @@ -286,7 +291,8 @@ def parse_cams(fbuf, integrated=False, label=None, map_variables=True): data.index = data.index - pd.Timedelta(days=1) if not integrated: # Convert radiation values from Wh/m2 to W/m2 - integrated_cols = [c for c in CAMS_INTEGRATED_COLUMNS if c in data.columns] # noqa + integrated_cols = [c for c in CAMS_RADIATION_INTEGRATED_COLUMNS + if c in data.columns] if time_step == '1M': time_delta = (pd.to_datetime(obs_period.str[1]) @@ -299,15 +305,15 @@ def parse_cams(fbuf, integrated=False, label=None, map_variables=True): TIME_STEPS_IN_HOURS[time_step]) if map_variables: - data = data.rename(columns=CAMS_VARIABLE_MAP) + data = data.rename(columns=CAMS_RADIATION_VARIABLE_MAP) return data, meta def read_cams(filename, integrated=False, label=None, map_variables=True): """ - Read a CAMS Radiation or McClear file. CAMS radiation and McClear is - described in [1]_ and [2]_, respectively. + Read a CAMS Radiation or McClear file into a pandas DataFrame. CAMS + radiation and McClear is described in [1]_ and [2]_, respectively. Parameters ---------- @@ -327,6 +333,7 @@ def read_cams(filename, integrated=False, label=None, map_variables=True): ------- data: pandas.DataFrame Timeseries data from CAMS Radiation or McClear + :func:`pvlib.iotools.get_cams` for fields meta: dict Metadata avaiable in the file. From 294c14a62694b05c03ec908adf2b13dcdd73d51b Mon Sep 17 00:00:00 2001 From: AdamRJensen Date: Sat, 27 Feb 2021 17:23:17 +0100 Subject: [PATCH 18/57] Convert print statements to warnings --- pvlib/iotools/cams.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pvlib/iotools/cams.py b/pvlib/iotools/cams.py index fc45989bf3..04bad612ee 100644 --- a/pvlib/iotools/cams.py +++ b/pvlib/iotools/cams.py @@ -6,6 +6,7 @@ import pandas as pd import requests import io +import warnings CAMS_RADIATION_INTEGRATED_COLUMNS = [ @@ -162,12 +163,12 @@ def get_cams_radiation(start_date, end_date, latitude, longitude, email, if time_step in TIME_STEPS_MAP.keys(): time_step_str = TIME_STEPS_MAP[time_step] else: - print('WARNING: time step not recognized, 1 hour time step used!') + warnings.warn('Time step not recognized, 1 hour time step used!') time_step, time_step_str = '1h', 'PT01H' if (verbose is True) & ((time_step != '1min') | (time_ref != 'UT')): verbose = False - print("Verbose mode only supports 1 min. UT time series!") + warnings.warn("Verbose mode only supports 1 min. UT time series!") # Format verbose variable to the required format: {'true', 'false'} verbose = str(verbose).lower() @@ -204,7 +205,7 @@ def get_cams_radiation(start_date, end_date, latitude, longitude, email, map_variables=map_variables) return data, meta else: - print('Error! File content type not recognized.') + warnings.warn('File content type not recognized.') def parse_cams(fbuf, integrated=False, label=None, map_variables=True): From d28143a0fc3dbc583c29a5cd92a2f8b452c2e60d Mon Sep 17 00:00:00 2001 From: AdamRJensen Date: Sat, 27 Feb 2021 18:32:15 +0100 Subject: [PATCH 19/57] Update function names --- pvlib/iotools/cams.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/pvlib/iotools/cams.py b/pvlib/iotools/cams.py index 04bad612ee..0e21778769 100644 --- a/pvlib/iotools/cams.py +++ b/pvlib/iotools/cams.py @@ -143,7 +143,7 @@ def get_cams_radiation(start_date, end_date, latitude, longitude, email, See Also -------- - pvlib.iotools.read_cams, pvlib.iotools.parse_cams + pvlib.iotools.read_cams_radiation, pvlib.iotools.parse_cams_radiation Raises ------ @@ -201,14 +201,14 @@ def get_cams_radiation(start_date, end_date, latitude, longitude, email, # Check if returned file is a csv data file elif res.headers['Content-Type'] == 'application/csv': fbuf = io.StringIO(res.content.decode('utf-8')) - data, meta = parse_cams(fbuf, integrated=integrated, label=label, + data, meta = parse_cams_radiation(fbuf, integrated=integrated, label=label, map_variables=map_variables) return data, meta else: warnings.warn('File content type not recognized.') -def parse_cams(fbuf, integrated=False, label=None, map_variables=True): +def parse_cams_radiation(fbuf, integrated=False, label=None, map_variables=True): """ Parse a file-like buffer with data in the format of a CAMS Radiation or McClear file. The CAMS servicess are described in [1]_ and [2]_. @@ -236,7 +236,7 @@ def parse_cams(fbuf, integrated=False, label=None, map_variables=True): See Also -------- - pvlib.iotools.read_cams, pvlib.iotools.get_cams + pvlib.iotools.read_cams_radiation, pvlib.iotools.get_cams_radiation References ---------- @@ -311,7 +311,8 @@ def parse_cams(fbuf, integrated=False, label=None, map_variables=True): return data, meta -def read_cams(filename, integrated=False, label=None, map_variables=True): +def read_cams_radiation(filename, integrated=False, label=None, + map_variables=True): """ Read a CAMS Radiation or McClear file into a pandas DataFrame. CAMS radiation and McClear is described in [1]_ and [2]_, respectively. @@ -340,7 +341,7 @@ def read_cams(filename, integrated=False, label=None, map_variables=True): See Also -------- - pvlib.iotools.parse_cams, pvlib.iotools.get_cams + pvlib.iotools.parse_cams_radiation, pvlib.iotools.get_cams_radiation References ---------- @@ -350,5 +351,5 @@ def read_cams(filename, integrated=False, label=None, map_variables=True): `_ """ with open(str(filename), 'r') as fbuf: - content = parse_cams(fbuf) + content = parse_cams_radiation(fbuf) return content From f2631849047a8faea7451fa741858abbbabb2f08 Mon Sep 17 00:00:00 2001 From: AdamRJensen Date: Sat, 27 Feb 2021 18:35:24 +0100 Subject: [PATCH 20/57] Fixed stickler --- pvlib/iotools/__init__.py | 4 +++- pvlib/iotools/cams.py | 5 +++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/pvlib/iotools/__init__.py b/pvlib/iotools/__init__.py index 65137d9be1..88f49f3a34 100644 --- a/pvlib/iotools/__init__.py +++ b/pvlib/iotools/__init__.py @@ -14,4 +14,6 @@ from pvlib.iotools.psm3 import parse_psm3 # noqa: F401 from pvlib.iotools.pvgis import get_pvgis_tmy, read_pvgis_tmy # noqa: F401 from pvlib.iotools.bsrn import read_bsrn # noqa: F401 -from pvlib.iotools.cams import get_cams_radiation, read_cams_radiation, parse_cams_radiation # noqa: F401 +from pvlib.iotools.cams import get_cams_radiation # noqa: F401 +from pvlib.iotools.cams import read_cams_radiation # noqa: F401 +from pvlib.iotools.cams import parse_cams_radiation # noqa: F401 diff --git a/pvlib/iotools/cams.py b/pvlib/iotools/cams.py index 0e21778769..e4dbe54b0f 100644 --- a/pvlib/iotools/cams.py +++ b/pvlib/iotools/cams.py @@ -201,8 +201,9 @@ def get_cams_radiation(start_date, end_date, latitude, longitude, email, # Check if returned file is a csv data file elif res.headers['Content-Type'] == 'application/csv': fbuf = io.StringIO(res.content.decode('utf-8')) - data, meta = parse_cams_radiation(fbuf, integrated=integrated, label=label, - map_variables=map_variables) + data, meta = parse_cams_radiation(fbuf, integrated=integrated, + label=label, + map_variables=map_variables) return data, meta else: warnings.warn('File content type not recognized.') From 9b26d91f69c5012744860ce01c77c4923a42ad5f Mon Sep 17 00:00:00 2001 From: AdamRJensen Date: Sat, 27 Feb 2021 18:36:11 +0100 Subject: [PATCH 21/57] Fixed stickler --- pvlib/iotools/cams.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pvlib/iotools/cams.py b/pvlib/iotools/cams.py index e4dbe54b0f..c8eb748620 100644 --- a/pvlib/iotools/cams.py +++ b/pvlib/iotools/cams.py @@ -209,7 +209,8 @@ def get_cams_radiation(start_date, end_date, latitude, longitude, email, warnings.warn('File content type not recognized.') -def parse_cams_radiation(fbuf, integrated=False, label=None, map_variables=True): +def parse_cams_radiation(fbuf, integrated=False, label=None, + map_variables=True): """ Parse a file-like buffer with data in the format of a CAMS Radiation or McClear file. The CAMS servicess are described in [1]_ and [2]_. From e3cb60d924ea8453cb9193704a52a8af3d207dd6 Mon Sep 17 00:00:00 2001 From: AdamRJensen Date: Sat, 27 Feb 2021 19:03:16 +0100 Subject: [PATCH 22/57] Improvements to documentation --- pvlib/iotools/cams.py | 60 +++++++++++++++++++++---------------------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/pvlib/iotools/cams.py b/pvlib/iotools/cams.py index c8eb748620..25e079cdd1 100644 --- a/pvlib/iotools/cams.py +++ b/pvlib/iotools/cams.py @@ -14,7 +14,7 @@ 'GHI', 'BHI', 'DHI', 'BNI', 'GHI no corr', 'BHI no corr', 'DHI no corr', 'BNI no corr'] -# Dictionary mapping CAMS McClear and Radiation variables to pvlib names +# Dictionary mapping CAMS Radiation and McClear variables to pvlib names CAMS_RADIATION_VARIABLE_MAP = { 'TOA': 'ghi_extra', 'Clear sky GHI': 'ghi_clear', @@ -29,7 +29,7 @@ } -# Dictionary mapping Python time steps to CAMS time step format +# Dictionary mapping time steps to CAMS time step format TIME_STEPS_MAP = {'1min': 'PT01M', '15min': 'PT15M', '1h': 'PT01H', '1d': 'P01D', '1M': 'P01M'} @@ -49,13 +49,16 @@ def get_cams_radiation(start_date, end_date, latitude, longitude, email, server='www.soda-is.com'): """ Retrieve time-series of radiation and/or clear-sky global, beam, and - diffuse radiation from CAMS [2]_ using the WGET service [3]_. + diffuse radiation from CAMS [1]_, [2]_ using the WGET service [3]_. Time coverage: 2004-01-01 to two days ago + Access: free, but requires registration, see [1]_ + Requests: max. 100 per day + Geographical coverage: Wordwide for CAMS McClear and -66° to 66° in both - latitude and longitude for CAMS Radiation + latitude and longitude for CAMS Radiation Parameters @@ -74,7 +77,7 @@ def get_cams_radiation(start_date, end_date, latitude, longitude, email, email: str Email address linked to a SoDa account service: {'mcclear', 'cams_radiation'} - Specify which whether to retrieve CAMS Radiation or McClear parameters + Specify whether to retrieve CAMS Radiation or McClear parameters time_step: str, {'1min', '15min', '1h', '1d', '1M'}, default: '1h' Time step of the time series, either 1 minute, 15 minute, hourly, daily, or monthly. @@ -84,14 +87,14 @@ def get_cams_radiation(start_date, end_date, latitude, longitude, email, Verbose mode outputs additional parameters (aerosols). Only avaiable for 1 minute and universal time. See [1] for parameter description. integrated: boolean, default False - Whether to return integrated irradiation values (Wh/m^2) from CAMS or - average irradiance values (W/m^2) as is more commonly used + Whether to return radiation parameters as integrated values (Wh/m^2) + or as average irradiance values (W/m^2) (pvlib preferred units) label: {‘right’, ‘left’}, default: None - Which bin edge label to label bucket with. The default is ‘left’ for - all frequency offsets except for ‘M’ which has a default of ‘right’. + Which bin edge label to label time-step with. The default is ‘left’ for + all time steps except for ‘1M’ which has a default of ‘right’. map_variables: bool, default: True - When true, renames columns of the Dataframe to pvlib variable names - where applicable. See variable CAMS_VARIABLE_MAP. + When true, renames columns of the DataFrame to pvlib variable names + where applicable. See variable CAMS_RADIATION_VARIABLE_MAP. server: str, default: 'www.soda-is.com' Main server (www.soda-is.com) or backup mirror server (pro.soda-is.com) @@ -113,7 +116,7 @@ def get_cams_radiation(start_date, end_date, latitude, longitude, email, Key, mapped key Format Description ======================= ====== ========================================== **Mapped field names are returned when the map_variables argument is True** - -------------------------------------------------------------------------- + --------------------------------------------------------------------------- Observation period str Beginning/end of time period TOA, ghi_extra float Horizontal radiation at top of atmosphere Clear sky GHI, ghi_clear float Clear sky global radiation on horizontal @@ -130,16 +133,13 @@ def get_cams_radiation(start_date, end_date, latitude, longitude, email, *Parameters only returned if service='cams_radiation'. For description of additional output parameters in verbose mode, see [1]_ and [2]_. - The returned units for the radiation parameters depends on the integrated - argument, i.e. integrated=False returns units of W/m2, whereas - integrated=True returns units of Wh/m2. - Note that it is recommended to specify the latitude and longitude to at least the fourth decimal place. Variables corresponding to standard pvlib variables are renamed, e.g. `sza` becomes `solar_zenith`. See the - `pvlib.iotools.cams.CAMS_VARIABLE_MAP` dict for the complete mapping. + `pvlib.iotools.cams.CAMS_RADIATION_VARIABLE_MAP` dict for the complete + mapping. See Also -------- @@ -181,7 +181,7 @@ def get_cams_radiation(start_date, end_date, latitude, longitude, email, end_date = end_date.strftime('%Y-%m-%d') email = email.replace('@', '%2540') # Format email address - service = 'get_{}'.format(service) # Format CAMS service string + service = 'get_{}'.format(service.lower()) # Format CAMS service string # Manual format the request url, due to uncommon usage of & and ; in url url = ("http://{}/service/wps?Service=WPS&Request=Execute&" @@ -220,14 +220,14 @@ def parse_cams_radiation(fbuf, integrated=False, label=None, fbuf: file-like object File-like object containing data to read. integrated: boolean, default False - Whether to return integrated irradiation values (Wh/m^2) from CAMS or - average irradiance values (W/m^2) as is more commonly used + Whether to return radiation parameters as integrated values (Wh/m^2) + or as average irradiance values (W/m^2) (pvlib preferred units) label: {‘right’, ‘left’}, default: None - Which bin edge label to label bucket with. The default is ‘left’ for - all frequency offsets except for ‘M’ which has a default of ‘right’. + Which bin edge label to label time-step with. The default is ‘left’ for + all time steps except for ‘1M’ which has a default of ‘right’. map_variables: bool, default: True When true, renames columns of the Dataframe to pvlib variable names - where applicable. See variable CAMS_VARIABLE_MAP. + where applicable. See variable CAMS_RADIATION_VARIABLE_MAP. Returns ------- @@ -317,27 +317,27 @@ def read_cams_radiation(filename, integrated=False, label=None, map_variables=True): """ Read a CAMS Radiation or McClear file into a pandas DataFrame. CAMS - radiation and McClear is described in [1]_ and [2]_, respectively. + radiation and McClear are described in [1]_ and [2]_, respectively. Parameters ---------- filename: str Filename of a file containing data to read. integrated: boolean, default False - Whether to return integrated irradiation values (Wh/m^2) from CAMS or - average irradiance values (W/m^2) as is more commonly used + Whether to return radiation parameters as integrated values (Wh/m^2) + or as average irradiance values (W/m^2) (pvlib preferred units) label: {‘right’, ‘left’}, default: None - Which bin edge label to label bucket with. The default is ‘left’ for - all frequency offsets except for ‘M’ which has a default of ‘right’. + Which bin edge label to label time-step with. The default is ‘left’ for + all time steps except for ‘1M’ which has a default of ‘right’. map_variables: bool, default: True When true, renames columns of the Dataframe to pvlib variable names - where applicable. See variable CAMS_VARIABLE_MAP. + where applicable. See variable CAMS_RADIATION_VARIABLE_MAP. Returns ------- data: pandas.DataFrame Timeseries data from CAMS Radiation or McClear - :func:`pvlib.iotools.get_cams` for fields + :func:`pvlib.iotools.get_cams_radiation` for fields meta: dict Metadata avaiable in the file. From af66bbe6ca483675153775d9dcf65548364f755b Mon Sep 17 00:00:00 2001 From: "Adam R. Jensen" <39184289+AdamRJensen@users.noreply.github.com> Date: Sat, 27 Feb 2021 23:00:49 +0100 Subject: [PATCH 23/57] Raise warning for unrecognized time step Co-authored-by: Will Holmgren --- pvlib/iotools/cams.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/pvlib/iotools/cams.py b/pvlib/iotools/cams.py index 25e079cdd1..0ed7ae9ebc 100644 --- a/pvlib/iotools/cams.py +++ b/pvlib/iotools/cams.py @@ -160,11 +160,10 @@ def get_cams_radiation(start_date, end_date, latitude, longitude, email, .. [3] `CAMS McClear Automatic Access `_ """ - if time_step in TIME_STEPS_MAP.keys(): + try: time_step_str = TIME_STEPS_MAP[time_step] - else: - warnings.warn('Time step not recognized, 1 hour time step used!') - time_step, time_step_str = '1h', 'PT01H' + except KeyError: + raise ValueError(f'Time step not recognized. Must be one of {list(TIME_STEPS_MAP.keys())}') if (verbose is True) & ((time_step != '1min') | (time_ref != 'UT')): verbose = False From 2949cb184e6e539dbcda9d45ebe71e7487701f08 Mon Sep 17 00:00:00 2001 From: AdamRJensen Date: Sun, 7 Mar 2021 00:26:09 +0100 Subject: [PATCH 24/57] Reworked URL formatting & comments on HTTP status --- pvlib/iotools/cams.py | 111 +++++++++++++++++++++++------------------- 1 file changed, 60 insertions(+), 51 deletions(-) diff --git a/pvlib/iotools/cams.py b/pvlib/iotools/cams.py index 0ed7ae9ebc..68097e2c87 100644 --- a/pvlib/iotools/cams.py +++ b/pvlib/iotools/cams.py @@ -28,7 +28,6 @@ 'sza': 'solar_zenith', } - # Dictionary mapping time steps to CAMS time step format TIME_STEPS_MAP = {'1min': 'PT01M', '15min': 'PT15M', '1h': 'PT01H', '1d': 'P01D', '1M': 'P01M'} @@ -43,7 +42,7 @@ def get_cams_radiation(start_date, end_date, latitude, longitude, email, - service='mcclear', altitude=None, time_step='1h', + identifier='mcclear', altitude=None, time_step='1h', time_ref='UT', verbose=False, integrated=False, label=None, map_variables=True, server='www.soda-is.com'): @@ -76,7 +75,7 @@ def get_cams_radiation(start_date, end_date, latitude, longitude, email, NASA SRTM database email: str Email address linked to a SoDa account - service: {'mcclear', 'cams_radiation'} + identifier: {'mcclear', 'cams_radiation'} Specify whether to retrieve CAMS Radiation or McClear parameters time_step: str, {'1min', '15min', '1h', '1d', '1M'}, default: '1h' Time step of the time series, either 1 minute, 15 minute, hourly, @@ -84,7 +83,7 @@ def get_cams_radiation(start_date, end_date, latitude, longitude, email, time_reference: str, {'UT', 'TST'}, default: 'UT' 'UT' (universal time) or 'TST' (True Solar Time) verbose: boolean, default: False - Verbose mode outputs additional parameters (aerosols). Only avaiable + Verbose mode outputs additional parameters (aerosols). Only available for 1 minute and universal time. See [1] for parameter description. integrated: boolean, default False Whether to return radiation parameters as integrated values (Wh/m^2) @@ -103,35 +102,35 @@ def get_cams_radiation(start_date, end_date, latitude, longitude, email, data: pandas.DataFrame Timeseries data, see Notes for columns meta: dict - Metadata for the requested time-series + Metadata of the requested time-series Notes ----- - In order to use the CAMS services, users must registre for a free SoDa - account using an email addres [1]_. + In order to use the CAMS services, users must register for a free SoDa + account using an email address [1]_. The returned data DataFrame includes the following fields: - ======================= ====== ========================================== - Key, mapped key Format Description - ======================= ====== ========================================== + ======================== ====== ========================================= + Key, mapped key Format Description + ======================== ====== ========================================= **Mapped field names are returned when the map_variables argument is True** --------------------------------------------------------------------------- - Observation period str Beginning/end of time period - TOA, ghi_extra float Horizontal radiation at top of atmosphere - Clear sky GHI, ghi_clear float Clear sky global radiation on horizontal - Clear sky BHI, bhi float Clear sky beam radiation on horizontal - Clear sky DHI, dhi_clear float Clear sky diffuse radiation on horizontal - Clear sky BNI, dni_clear float Clear sky beam radiation normal to sun - GHI, ghi* float Global horizontal radiation - BHI, bhi* float Beam (direct) radiation on horizontal - DHI, dhi* float Diffuse horizontal radiation - BNI, dni* float Beam (direct) radiation normal to the sun - Reliability* float Fraction of reliable data in summarization - ======================= ====== ========================================== - - *Parameters only returned if service='cams_radiation'. For description of - additional output parameters in verbose mode, see [1]_ and [2]_. + Observation period str Beginning/end of time period + TOA, ghi_extra float Horizontal radiation at top of atmosphere + Clear sky GHI, ghi_clear float Clear sky global radiation on horizontal + Clear sky BHI, bhi float Clear sky beam radiation on horizontal + Clear sky DHI, dhi_clear float Clear sky diffuse radiation on horizontal + Clear sky BNI, dni_clear float Clear sky beam radiation normal to sun + GHI, ghi* float Global horizontal radiation + BHI, bhi* float Beam (direct) radiation on horizontal + DHI, dhi* float Diffuse horizontal radiation + BNI, dni* float Beam (direct) radiation normal to the sun + Reliability* float Reliable data fraction in summarization + ======================== ====== ========================================= + + *Parameters only returned if identifier='cams_radiation'. For description + of additional output parameters in verbose mode, see [1]_ and [2]_. Note that it is recommended to specify the latitude and longitude to at least the fourth decimal place. @@ -149,7 +148,7 @@ def get_cams_radiation(start_date, end_date, latitude, longitude, email, ------ requests.HTTPError If the request is invalid, then an XML file is returned by the CAMS - service and the error message will be raised as an expcetion. + service and the error message will be raised as an exception. References ---------- @@ -163,7 +162,8 @@ def get_cams_radiation(start_date, end_date, latitude, longitude, email, try: time_step_str = TIME_STEPS_MAP[time_step] except KeyError: - raise ValueError(f'Time step not recognized. Must be one of {list(TIME_STEPS_MAP.keys())}') + raise ValueError(f'Time step not recognized. Must be one of ' + f'{list(TIME_STEPS_MAP.keys())}') if (verbose is True) & ((time_step != '1min') | (time_ref != 'UT')): verbose = False @@ -180,32 +180,41 @@ def get_cams_radiation(start_date, end_date, latitude, longitude, email, end_date = end_date.strftime('%Y-%m-%d') email = email.replace('@', '%2540') # Format email address - service = 'get_{}'.format(service.lower()) # Format CAMS service string - - # Manual format the request url, due to uncommon usage of & and ; in url - url = ("http://{}/service/wps?Service=WPS&Request=Execute&" - "Identifier={}&version=1.0.0&RawDataOutput=irradiation&" - "DataInputs=latitude={};longitude={};altitude={};" - "date_begin={};date_end={};time_ref={};summarization={};" - "username={};verbose={}" - ).format(server, service, latitude, longitude, altitude, start_date, - end_date, time_ref, time_step_str, email, verbose) - - res = requests.get(url) - - # Invalid requests returns helpful XML error message + identifier = 'get_{}'.format(identifier.lower()) # Format identifier str + + base_url = f"http://{server}/service/wps" + + data_inputs = ( + f"latitude={latitude};longitude={longitude};altitude={altitude};" + f"date_begin={start_date};date_end={end_date};time_ref={time_ref};" + f"summarization={time_step_str};username={email};verbose={verbose}") + + params = {'Service': 'WPS', + 'Request': 'Execute', + 'Identifier': identifier, + 'version': '1.0.0', + 'RawDataOutput': 'irradiation', + } + + # The DataInputs parameter of the URL has to be manually formatted and + # added to the base URL as it contains sub-parameters seperated by + # semi-colons, which gets incorrectly formatted by the requests function + # if passed using the params argument. + res = requests.get(base_url + '?DataInputs=' + data_inputs, params=params) + + # Invalid requests returns an XML error message and the HTTP staus code 200 + # as if the request was successful. Therefore, errors cannot be handled + # automatic (e.g. res.raise_for_status()) and errors are handled manually if res.headers['Content-Type'] == 'application/xml': errors = res.text.split('ows:ExceptionText')[1][1:-2] raise requests.HTTPError(errors, response=res) - # Check if returned file is a csv data file + # Successful requests returns a csv data file elif res.headers['Content-Type'] == 'application/csv': fbuf = io.StringIO(res.content.decode('utf-8')) data, meta = parse_cams_radiation(fbuf, integrated=integrated, label=label, map_variables=map_variables) return data, meta - else: - warnings.warn('File content type not recognized.') def parse_cams_radiation(fbuf, integrated=False, label=None, @@ -257,13 +266,13 @@ def parse_cams_radiation(fbuf, integrated=False, label=None, elif ': ' in line: meta[line.split(': ')[0].lstrip('# ')] = line.split(': ')[1] - # Convert the latitude, longitude, and altitude from strings to floats - meta['Latitude (positive North, ISO 19115)'] = \ - float(meta['Latitude (positive North, ISO 19115)']) - meta['Longitude (positive East, ISO 19115)'] = \ - float(meta['Longitude (positive East, ISO 19115)']) - meta['Altitude (m)'] = float(meta['Altitude (m)']) - meta['Unit for radiation'] = {True: 'Wh/m2', False: 'W/m2'}[integrated] + # Convert latitude, longitude, and altitude values from strings to floats + for k_old in list(meta.keys()): + k_new = k_old.lstrip().split(' ')[0].lower() + if k_new in ['latitude', 'longitude', 'altitude']: + meta[k_new] = float(meta.pop(k_old)) + + meta['radiation_unit'] = {True: 'Wh/m2', False: 'W/m2'}[integrated] # Determine the time_step from the meta-data dictionary time_step = SUMMATION_PERIOD_TO_TIME_STEP[ From 19fcdb897b1028668d1e9efdb13b0eba300f8870 Mon Sep 17 00:00:00 2001 From: AdamRJensen Date: Sun, 7 Mar 2021 00:29:42 +0100 Subject: [PATCH 25/57] Fixed stickler --- pvlib/iotools/cams.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pvlib/iotools/cams.py b/pvlib/iotools/cams.py index 68097e2c87..7bf3d212e0 100644 --- a/pvlib/iotools/cams.py +++ b/pvlib/iotools/cams.py @@ -270,8 +270,8 @@ def parse_cams_radiation(fbuf, integrated=False, label=None, for k_old in list(meta.keys()): k_new = k_old.lstrip().split(' ')[0].lower() if k_new in ['latitude', 'longitude', 'altitude']: - meta[k_new] = float(meta.pop(k_old)) - + meta[k_new] = float(meta.pop(k_old)) + meta['radiation_unit'] = {True: 'Wh/m2', False: 'W/m2'}[integrated] # Determine the time_step from the meta-data dictionary From 328c86e1ece7388212bae1703fc12deea8524945 Mon Sep 17 00:00:00 2001 From: AdamRJensen Date: Sun, 7 Mar 2021 18:16:33 +0100 Subject: [PATCH 26/57] Add tests for cams.read_cams_radiation --- pvlib/data/cams_mcclear_1min_verbose.csv | 56 -------- pvlib/data/cams_mcclear_monthly.csv | 8 -- pvlib/data/cams_radiation_1min_verbose.csv | 72 ++++++++++ pvlib/data/cams_radiation_monthly.csv | 47 +++++++ pvlib/iotools/cams.py | 10 +- pvlib/tests/iotools/test_cams.py | 147 +++++++++++++++++++++ 6 files changed, 271 insertions(+), 69 deletions(-) create mode 100644 pvlib/data/cams_radiation_1min_verbose.csv create mode 100644 pvlib/data/cams_radiation_monthly.csv create mode 100644 pvlib/tests/iotools/test_cams.py diff --git a/pvlib/data/cams_mcclear_1min_verbose.csv b/pvlib/data/cams_mcclear_1min_verbose.csv index 9d192d9dfe..0bfb037b08 100644 --- a/pvlib/data/cams_mcclear_1min_verbose.csv +++ b/pvlib/data/cams_mcclear_1min_verbose.csv @@ -58,59 +58,3 @@ 2020-06-01T12:01:00.0/2020-06-01T12:02:00.0;18.0584;14.1311;12.5484;1.5827;15.3343;35.0828;0.9723;341.0223;17.8020;0.0065;0.0067;0.0008;0.0215;0.0253;0.0087;0.0022;nan;-1;0.1668;0.0912;0.0267;0.1359 2020-06-01T12:02:00.0/2020-06-01T12:03:00.0;18.0467;14.1204;12.5372;1.5831;15.3306;35.1357;0.9723;341.0224;17.8079;0.0065;0.0067;0.0008;0.0216;0.0253;0.0087;0.0022;nan;-1;0.1668;0.0912;0.0267;0.1359 2020-06-01T12:03:00.0/2020-06-01T12:04:00.0;18.0348;14.1094;12.5259;1.5835;15.3269;35.1896;0.9723;341.0226;17.8137;0.0065;0.0067;0.0008;0.0217;0.0253;0.0087;0.0022;nan;-1;0.1668;0.0912;0.0267;0.1359 -2020-06-01T12:04:00.0/2020-06-01T12:05:00.0;18.0226;14.0983;12.5144;1.5839;15.3231;35.2442;0.9723;341.0228;17.8196;0.0064;0.0067;0.0008;0.0218;0.0254;0.0087;0.0022;nan;-1;0.1668;0.0912;0.0267;0.1360 -2020-06-01T12:05:00.0/2020-06-01T12:06:00.0;18.0103;14.0869;12.5026;1.5843;15.3193;35.2998;0.9723;341.0229;17.8254;0.0064;0.0067;0.0008;0.0218;0.0254;0.0087;0.0022;nan;-1;0.1668;0.0912;0.0267;0.1360 -2020-06-01T12:06:00.0/2020-06-01T12:07:00.0;17.9977;14.0754;12.4908;1.5846;15.3154;35.3563;0.9723;341.0231;17.8313;0.0064;0.0067;0.0008;0.0219;0.0254;0.0087;0.0022;nan;-1;0.1668;0.0912;0.0267;0.1360 -2020-06-01T12:07:00.0/2020-06-01T12:08:00.0;17.9849;14.0637;12.4787;1.5850;15.3114;35.4136;0.9723;341.0233;17.8371;0.0064;0.0066;0.0008;0.0220;0.0255;0.0087;0.0022;nan;-1;0.1668;0.0912;0.0267;0.1360 -2020-06-01T12:08:00.0/2020-06-01T12:09:00.0;17.9719;14.0518;12.4664;1.5854;15.3075;35.4717;0.9723;341.0234;17.8430;0.0064;0.0066;0.0008;0.0220;0.0255;0.0087;0.0022;nan;-1;0.1668;0.0912;0.0267;0.1360 -2020-06-01T12:09:00.0/2020-06-01T12:10:00.0;17.9587;14.0397;12.4540;1.5857;15.3035;35.5308;0.9723;341.0236;17.8488;0.0064;0.0066;0.0008;0.0221;0.0255;0.0087;0.0022;nan;-1;0.1668;0.0912;0.0267;0.1361 -2020-06-01T12:10:00.0/2020-06-01T12:11:00.0;17.9453;14.0274;12.4414;1.5860;15.2994;35.5907;0.9724;341.0238;17.8547;0.0064;0.0066;0.0008;0.0222;0.0256;0.0087;0.0022;nan;-1;0.1668;0.0912;0.0267;0.1361 -2020-06-01T12:11:00.0/2020-06-01T12:12:00.0;17.9316;14.0150;12.4286;1.5864;15.2953;35.6514;0.9724;341.0239;17.8605;0.0064;0.0066;0.0008;0.0223;0.0256;0.0087;0.0022;nan;-1;0.1668;0.0912;0.0267;0.1361 -2020-06-01T12:12:00.0/2020-06-01T12:13:00.0;17.9178;14.0023;12.4157;1.5867;15.2911;35.7130;0.9724;341.0241;17.8664;0.0063;0.0066;0.0008;0.0223;0.0256;0.0087;0.0022;nan;-1;0.1668;0.0912;0.0267;0.1361 -2020-06-01T12:13:00.0/2020-06-01T12:14:00.0;17.9037;13.9895;12.4025;1.5870;15.2869;35.7754;0.9724;341.0243;17.8722;0.0063;0.0066;0.0008;0.0224;0.0257;0.0087;0.0022;nan;-1;0.1668;0.0912;0.0267;0.1361 -2020-06-01T12:14:00.0/2020-06-01T12:15:00.0;17.8895;13.9765;12.3892;1.5873;15.2827;35.8387;0.9724;341.0244;17.8781;0.0063;0.0066;0.0008;0.0225;0.0257;0.0087;0.0022;nan;-1;0.1668;0.0912;0.0267;0.1362 -2020-06-01T12:15:00.0/2020-06-01T12:16:00.0;17.8750;13.9633;12.3757;1.5876;15.2784;35.9028;0.9724;341.0246;17.8839;0.0063;0.0065;0.0008;0.0225;0.0258;0.0087;0.0022;nan;-1;0.1668;0.0912;0.0267;0.1362 -2020-06-01T12:16:00.0/2020-06-01T12:17:00.0;17.8603;13.9499;12.3620;1.5879;15.2741;35.9677;0.9724;341.0247;17.8898;0.0063;0.0065;0.0008;0.0226;0.0258;0.0087;0.0022;nan;-1;0.1668;0.0912;0.0267;0.1362 -2020-06-01T12:17:00.0/2020-06-01T12:18:00.0;17.8454;13.9363;12.3482;1.5882;15.2697;36.0335;0.9724;341.0249;17.8956;0.0063;0.0065;0.0008;0.0227;0.0258;0.0087;0.0022;nan;-1;0.1668;0.0912;0.0267;0.1362 -2020-06-01T12:18:00.0/2020-06-01T12:19:00.0;17.8303;13.9226;12.3342;1.5884;15.2652;36.1000;0.9724;341.0251;17.9015;0.0063;0.0065;0.0008;0.0227;0.0259;0.0087;0.0022;nan;-1;0.1668;0.0912;0.0267;0.1362 -2020-06-01T12:19:00.0/2020-06-01T12:20:00.0;17.8150;13.9087;12.3200;1.5887;15.2608;36.1674;0.9724;341.0252;17.9073;0.0063;0.0065;0.0008;0.0228;0.0259;0.0087;0.0022;nan;-1;0.1668;0.0912;0.0267;0.1363 -2020-06-01T12:20:00.0/2020-06-01T12:21:00.0;17.7995;13.8945;12.3056;1.5890;15.2562;36.2356;0.9724;341.0254;17.9132;0.0062;0.0065;0.0008;0.0229;0.0259;0.0087;0.0022;nan;-1;0.1668;0.0912;0.0267;0.1363 -2020-06-01T12:21:00.0/2020-06-01T12:22:00.0;17.7838;13.8802;12.2910;1.5892;15.2517;36.3046;0.9724;341.0256;17.9190;0.0062;0.0065;0.0008;0.0230;0.0260;0.0087;0.0022;nan;-1;0.1668;0.0912;0.0267;0.1363 -2020-06-01T12:22:00.0/2020-06-01T12:23:00.0;17.7679;13.8657;12.2763;1.5894;15.2470;36.3744;0.9724;341.0257;17.9249;0.0062;0.0064;0.0008;0.0230;0.0260;0.0087;0.0022;nan;-1;0.1668;0.0912;0.0267;0.1363 -2020-06-01T12:23:00.0/2020-06-01T12:24:00.0;17.7517;13.8511;12.2614;1.5897;15.2424;36.4449;0.9724;341.0259;17.9307;0.0062;0.0064;0.0008;0.0231;0.0260;0.0087;0.0022;nan;-1;0.1668;0.0912;0.0267;0.1364 -2020-06-01T12:24:00.0/2020-06-01T12:25:00.0;17.7354;13.8362;12.2463;1.5899;15.2377;36.5163;0.9724;341.0261;17.9366;0.0062;0.0064;0.0008;0.0232;0.0261;0.0087;0.0022;nan;-1;0.1668;0.0912;0.0267;0.1364 -2020-06-01T12:25:00.0/2020-06-01T12:26:00.0;17.7188;13.8212;12.2311;1.5901;15.2329;36.5884;0.9724;341.0262;17.9424;0.0062;0.0064;0.0008;0.0232;0.0261;0.0087;0.0022;nan;-1;0.1668;0.0912;0.0267;0.1364 -2020-06-01T12:26:00.0/2020-06-01T12:27:00.0;17.7021;13.8060;12.2156;1.5903;15.2281;36.6613;0.9724;341.0264;17.9483;0.0062;0.0064;0.0008;0.0233;0.0261;0.0087;0.0022;nan;-1;0.1668;0.0912;0.0267;0.1364 -2020-06-01T12:27:00.0/2020-06-01T12:28:00.0;17.6851;13.7905;12.2000;1.5905;15.2232;36.7350;0.9724;341.0266;17.9541;0.0062;0.0064;0.0008;0.0234;0.0262;0.0087;0.0022;nan;-1;0.1668;0.0912;0.0267;0.1365 -2020-06-01T12:28:00.0/2020-06-01T12:29:00.0;17.6680;13.7750;12.1843;1.5907;15.2183;36.8094;0.9724;341.0267;17.9600;0.0061;0.0064;0.0008;0.0234;0.0262;0.0087;0.0022;nan;-1;0.1668;0.0912;0.0267;0.1365 -2020-06-01T12:29:00.0/2020-06-01T12:30:00.0;17.6506;13.7592;12.1683;1.5909;15.2133;36.8846;0.9724;341.0269;17.9658;0.0061;0.0063;0.0008;0.0235;0.0263;0.0087;0.0022;nan;-1;0.1668;0.0912;0.0267;0.1365 -2020-06-01T12:30:00.0/2020-06-01T12:31:00.0;17.6330;13.7433;12.1522;1.5910;15.2083;36.9606;0.9724;341.0271;17.9717;0.0061;0.0063;0.0008;0.0236;0.0263;0.0087;0.0022;nan;-1;0.1668;0.0912;0.0267;0.1366 -2020-06-01T12:31:00.0/2020-06-01T12:32:00.0;17.6152;13.7271;12.1359;1.5912;15.2033;37.0373;0.9724;341.0272;17.9775;0.0061;0.0063;0.0008;0.0237;0.0263;0.0087;0.0022;nan;-1;0.1668;0.0912;0.0267;0.1366 -2020-06-01T12:32:00.0/2020-06-01T12:33:00.0;17.5972;13.7108;12.1195;1.5914;15.1982;37.1147;0.9724;341.0274;17.9834;0.0061;0.0063;0.0008;0.0237;0.0264;0.0087;0.0022;nan;-1;0.1668;0.0912;0.0267;0.1366 -2020-06-01T12:33:00.0/2020-06-01T12:34:00.0;17.5790;13.6943;12.1028;1.5915;15.1930;37.1929;0.9724;341.0276;17.9892;0.0061;0.0063;0.0008;0.0238;0.0264;0.0087;0.0022;nan;-1;0.1668;0.0912;0.0267;0.1366 -2020-06-01T12:34:00.0/2020-06-01T12:35:00.0;17.5606;13.6777;12.0860;1.5917;15.1878;37.2718;0.9724;341.0277;17.9951;0.0061;0.0063;0.0008;0.0239;0.0264;0.0087;0.0022;nan;-1;0.1668;0.0912;0.0267;0.1367 -2020-06-01T12:35:00.0/2020-06-01T12:36:00.0;17.5420;13.6608;12.0691;1.5918;15.1826;37.3515;0.9724;341.0279;18.0010;0.0061;0.0063;0.0007;0.0239;0.0265;0.0087;0.0022;nan;-1;0.1668;0.0912;0.0267;0.1367 -2020-06-01T12:36:00.0/2020-06-01T12:37:00.0;17.5233;13.6438;12.0519;1.5919;15.1773;37.4318;0.9724;341.0280;18.0068;0.0060;0.0063;0.0007;0.0240;0.0265;0.0087;0.0022;nan;-1;0.1668;0.0912;0.0267;0.1367 -2020-06-01T12:37:00.0/2020-06-01T12:38:00.0;17.5042;13.6266;12.0346;1.5920;15.1719;37.5129;0.9724;341.0282;18.0127;0.0060;0.0062;0.0007;0.0241;0.0265;0.0087;0.0022;nan;-1;0.1668;0.0912;0.0267;0.1368 -2020-06-01T12:38:00.0/2020-06-01T12:39:00.0;17.4850;13.6092;12.0171;1.5921;15.1665;37.5947;0.9724;341.0284;18.0185;0.0060;0.0062;0.0007;0.0242;0.0266;0.0087;0.0022;nan;-1;0.1668;0.0912;0.0267;0.1368 -2020-06-01T12:39:00.0/2020-06-01T12:40:00.0;17.4656;13.5917;11.9995;1.5922;15.1611;37.6772;0.9724;341.0285;18.0244;0.0060;0.0062;0.0007;0.0242;0.0266;0.0087;0.0022;nan;-1;0.1668;0.0912;0.0267;0.1368 -2020-06-01T12:40:00.0/2020-06-01T12:41:00.0;17.4460;13.5739;11.9816;1.5923;15.1555;37.7604;0.9724;341.0287;18.0302;0.0060;0.0062;0.0007;0.0243;0.0266;0.0087;0.0022;nan;-1;0.1668;0.0912;0.0267;0.1369 -2020-06-01T12:41:00.0/2020-06-01T12:42:00.0;17.4262;13.5560;11.9637;1.5924;15.1500;37.8443;0.9724;341.0289;18.0361;0.0060;0.0062;0.0007;0.0244;0.0267;0.0087;0.0022;nan;-1;0.1668;0.0912;0.0267;0.1369 -2020-06-01T12:42:00.0/2020-06-01T12:43:00.0;17.4062;13.5379;11.9455;1.5924;15.1444;37.9289;0.9724;341.0290;18.0419;0.0060;0.0062;0.0007;0.0244;0.0267;0.0087;0.0022;nan;-1;0.1668;0.0912;0.0267;0.1369 -2020-06-01T12:43:00.0/2020-06-01T12:44:00.0;17.3860;13.5197;11.9272;1.5925;15.1387;38.0141;0.9724;341.0292;18.0478;0.0060;0.0062;0.0007;0.0245;0.0267;0.0087;0.0022;nan;-1;0.1668;0.0912;0.0267;0.1370 -2020-06-01T12:44:00.0/2020-06-01T12:45:00.0;17.3656;13.5012;11.9087;1.5925;15.1330;38.1001;0.9724;341.0294;18.0536;0.0059;0.0061;0.0007;0.0246;0.0268;0.0087;0.0022;nan;-1;0.1668;0.0912;0.0267;0.1370 -2020-06-01T12:45:00.0/2020-06-01T12:46:00.0;17.3450;13.4826;11.8900;1.5926;15.1273;38.1867;0.9724;341.0295;18.0595;0.0059;0.0061;0.0007;0.0246;0.0268;0.0087;0.0022;nan;-1;0.1668;0.0912;0.0267;0.1370 -2020-06-01T12:46:00.0/2020-06-01T12:47:00.0;17.3242;13.4638;11.8712;1.5926;15.1215;38.2740;0.9724;341.0297;18.0653;0.0059;0.0061;0.0007;0.0247;0.0269;0.0087;0.0022;nan;-1;0.1668;0.0912;0.0267;0.1371 -2020-06-01T12:47:00.0/2020-06-01T12:48:00.0;17.3032;13.4449;11.8522;1.5926;15.1156;38.3620;0.9724;341.0299;18.0712;0.0059;0.0061;0.0007;0.0248;0.0269;0.0087;0.0022;nan;-1;0.1668;0.0912;0.0267;0.1371 -2020-06-01T12:48:00.0/2020-06-01T12:49:00.0;17.2820;13.4257;11.8331;1.5927;15.1097;38.4506;0.9724;341.0300;18.0770;0.0059;0.0061;0.0007;0.0249;0.0269;0.0087;0.0022;nan;-1;0.1668;0.0912;0.0267;0.1371 -2020-06-01T12:49:00.0/2020-06-01T12:50:00.0;17.2605;13.4064;11.8137;1.5927;15.1037;38.5398;0.9724;341.0302;18.0829;0.0059;0.0061;0.0007;0.0249;0.0270;0.0087;0.0022;nan;-1;0.1668;0.0912;0.0267;0.1372 -2020-06-01T12:50:00.0/2020-06-01T12:51:00.0;17.2390;13.3869;11.7943;1.5927;15.0977;38.6297;0.9724;341.0304;18.0887;0.0059;0.0061;0.0007;0.0250;0.0270;0.0087;0.0022;nan;-1;0.1668;0.0912;0.0267;0.1372 -2020-06-01T12:51:00.0/2020-06-01T12:52:00.0;17.2171;13.3673;11.7746;1.5926;15.0916;38.7203;0.9724;341.0305;18.0946;0.0059;0.0060;0.0007;0.0251;0.0270;0.0087;0.0022;nan;-1;0.1668;0.0912;0.0267;0.1372 -2020-06-01T12:52:00.0/2020-06-01T12:53:00.0;17.1952;13.3474;11.7548;1.5926;15.0855;38.8115;0.9724;341.0307;18.1004;0.0058;0.0060;0.0007;0.0251;0.0271;0.0087;0.0022;nan;-1;0.1668;0.0912;0.0267;0.1373 -2020-06-01T12:53:00.0/2020-06-01T12:54:00.0;17.1730;13.3274;11.7348;1.5926;15.0793;38.9033;0.9724;341.0308;18.1063;0.0058;0.0060;0.0007;0.0252;0.0271;0.0087;0.0022;nan;-1;0.1668;0.0912;0.0267;0.1373 -2020-06-01T12:54:00.0/2020-06-01T12:55:00.0;17.1506;13.3072;11.7147;1.5926;15.0731;38.9957;0.9724;341.0310;18.1121;0.0058;0.0060;0.0007;0.0253;0.0271;0.0087;0.0022;nan;-1;0.1668;0.0912;0.0267;0.1374 -2020-06-01T12:55:00.0/2020-06-01T12:56:00.0;17.1280;13.2869;11.6944;1.5925;15.0668;39.0888;0.9724;341.0312;18.1180;0.0058;0.0060;0.0007;0.0253;0.0272;0.0087;0.0022;nan;-1;0.1668;0.0912;0.0267;0.1374 -2020-06-01T12:56:00.0/2020-06-01T12:57:00.0;17.1052;13.2664;11.6739;1.5924;15.0605;39.1825;0.9724;341.0313;18.1238;0.0058;0.0060;0.0007;0.0254;0.0272;0.0086;0.0022;nan;-1;0.1668;0.0912;0.0267;0.1374 -2020-06-01T12:57:00.0/2020-06-01T12:58:00.0;17.0822;13.2457;11.6533;1.5924;15.0541;39.2768;0.9724;341.0315;18.1297;0.0058;0.0060;0.0007;0.0255;0.0272;0.0086;0.0022;nan;-1;0.1668;0.0912;0.0267;0.1375 -2020-06-01T12:58:00.0/2020-06-01T12:59:00.0;17.0591;13.2248;11.6325;1.5923;15.0476;39.3717;0.9724;341.0317;18.1355;0.0058;0.0060;0.0007;0.0256;0.0273;0.0086;0.0022;nan;-1;0.1668;0.0912;0.0267;0.1375 -2020-06-01T12:59:00.0/2020-06-01T13:00:00.0;17.0357;13.2038;11.6116;1.5922;15.0411;39.4672;0.9724;341.0318;18.1414;0.0058;0.0059;0.0007;0.0256;0.0273;0.0086;0.0022;nan;-1;0.1668;0.0912;0.0267;0.1376 diff --git a/pvlib/data/cams_mcclear_monthly.csv b/pvlib/data/cams_mcclear_monthly.csv index f490cf2792..70dfa99333 100644 --- a/pvlib/data/cams_mcclear_monthly.csv +++ b/pvlib/data/cams_mcclear_monthly.csv @@ -40,11 +40,3 @@ 2020-02-01T00:00:00.0/2020-03-01T00:00:00.0;91338.5234;59010.3047;40636.3008;18374.0020;140930.5781 2020-03-01T00:00:00.0/2020-04-01T00:00:00.0;172855.2031;121402.9141;93124.6250;28278.2891;228798.9062 2020-04-01T00:00:00.0/2020-05-01T00:00:00.0;248215.0312;180546.1406;142470.4688;38075.6680;279122.9375 -2020-05-01T00:00:00.0/2020-06-01T00:00:00.0;323612.6562;236514.2969;192534.5156;43979.7695;333939.2500 -2020-06-01T00:00:00.0/2020-07-01T00:00:00.0;342520.5938;243592.3438;190944.2812;52648.0664;314883.6875 -2020-07-01T00:00:00.0/2020-08-01T00:00:00.0;336242.4375;241529.2656;195424.1250;46105.1484;331152.5625 -2020-08-01T00:00:00.0/2020-09-01T00:00:00.0;277367.4375;194389.0000;150684.6250;43704.3633;278330.5312 -2020-09-01T00:00:00.0/2020-10-01T00:00:00.0;191782.4062;128731.1172;93966.3203;34764.7969;206941.1250 -2020-10-01T00:00:00.0/2020-11-01T00:00:00.0;119269.3047;77265.0312;55354.8828;21910.1484;168171.4062 -2020-11-01T00:00:00.0/2020-12-01T00:00:00.0;59529.1133;35376.3125;23355.3477;12020.9658;110117.4375 -2020-12-01T00:00:00.0/2021-01-01T00:00:00.0;37984.8984;20478.5742;12214.5166;8264.0566;80236.4766 diff --git a/pvlib/data/cams_radiation_1min_verbose.csv b/pvlib/data/cams_radiation_1min_verbose.csv new file mode 100644 index 0000000000..3686c86f4e --- /dev/null +++ b/pvlib/data/cams_radiation_1min_verbose.csv @@ -0,0 +1,72 @@ +# Coding: utf-8 +# File format version: 5 +# Title: CAMS Radiation Service v3.2 all-sky irradiation (derived from satellite data). +# Content: A time-series of solar radiation received on horizontal plane and plane always normal to the sun rays at ground level. +# Returns the global, beam and diffuse irradiations integrated over a selected time step, +# for a selected location (Meteosat Second Generation satellite coverage) and a selected period. +# The research leading to these results has received funding from the European Union within the Copernicus programme. +# Provider: MINES ParisTech (France) +# More information at: http://www.soda-pro.com/web-services/radiation/cams-radiation-service +# Date begin (ISO 8601): 2020-06-01T00:00:00.0 +# Date end (ISO 8601): 2020-06-02T00:00:00.0 +# Latitude (positive North, ISO 19115): 55.7906 +# Longitude (positive East, ISO 19115): 12.5251 +# Altitude (m): 39.00 +# Elevation of CAMS cell (m): 28.64 +# Time reference: Universal time (UT) +# +# Encoding partly from D2.8.III.13-14 INSPIRE Data Specification on Atmospheric Conditions and Meteorological Geographical Features - Technical Guidelines (2013-12-10) and CF (Climate and Forecast) metadata (2013-11-11) +# CF Standard Names registry of ObservablePropertyValue +# http://cfconventions.org/Data/cf-standard-names/27/build/cf-standard-name-table.html +# urn:x-inspire:specification:DS-AC-MF:observable-property-name:cf-standard-name:1.6 +# ObservableProperty +# basePhenomenon:"integral_of_surface_downwelling_shortwave_flux_in_air_sky_wrt_time" +# uom:"Wh m-2" [unit] +# StatisticalMeasure +# statisticalFunction: "sum" +# Summarization (integration) period: 0 year 0 month 0 day 0 h 1 min 0 s +# noValue: nan +# File generated on: 2021-03-07 +# +# Columns: +# 1. Observation period (ISO 8601) +# 2. TOA. Irradiation on horizontal plane at the top of atmosphere (Wh/m2) +# 3. Clear sky GHI. Clear sky global irradiation on horizontal plane at ground level (Wh/m2) +# 4. Clear sky BHI. Clear sky beam irradiation on horizontal plane at ground level (Wh/m2) +# 5. Clear sky DHI. Clear sky diffuse irradiation on horizontal plane at ground level (Wh/m2) +# 6. Clear sky BNI. Clear sky beam irradiation on mobile plane following the sun at normal incidence (Wh/m2) +# 7. GHI. Global irradiation on horizontal plane at ground level (Wh/m2) +# 8. BHI. Beam irradiation on horizontal plane at ground level (Wh/m2) +# 9. DHI. Diffuse irradiation on horizontal plane at ground level (Wh/m2) +#10. BNI. Beam irradiation on mobile plane following the sun at normal incidence (Wh/m2) +#11. Reliability. Proportion of reliable data in the summarization (0-1) +#12. sza. Solar zenithal angle for the middle of the summarization (deg) +#13. summer/winter split. 1.0 means summer, 0.0 means winter +#14. tco3. Total column content of ozone (Dobson unit) +#15. tcwv. Total column content of water vapour (kg/m2) +#16. AOD BC. Partial aerosol optical depth at 550 nm for black carbon +#17. AOD DU. Partial aerosol optical depth at 550 nm for dust +#18. AOD SS. Partial aerosol optical depth at 550 nm for sea salt +#19. AOD OR. Partial aerosol optical depth at 550 nm for organic matter +#20. AOD SU. Partial aerosol optical depth at 550 nm for sulphate +#21. AOD NI. Partial aerosol optical depth at 550 nm for nitrate +#22. AOD AM. Partial aerosol optical depth at 550 nm for ammonium +#23. alpha. Angstroem coefficient for aerosol +#24. Aerosol type. Obsolete (value is always -1) +#25. fiso. MODIS-like BRDF parameter fiso +#26. fvol. MODIS-like BRDF parameter fvol +#27. fgeo. MODIS-like BRDF parameter fgeo +#28. albedo. Ground albedo +#29. Cloud optical depth (value of the nearest acquisition time of the pixel) +#30. Cloud coverage of the pixel (percentage from 0 to 100, value of the nearest acquisition time of the pixel) -1=no value +#31. Cloud type (value of the nearest acquisition time of the pixel) -1=no value 0=no clouds 5=low-level cloud 6=medium-level cloud 7=high-level cloud 8=thin cloud +#32. GHI no corr. Global irradiation without bias correction on horizontal plane at ground level (Wh/m2) +#33. BHI no corr. Beam irradiation without bias correction on horizontal plane at ground level (Wh/m2) +#34. DHI no corr. Diffuse irradiation without bias correction on horizontal plane at ground level (Wh/m2) +#35. BNI no corr. Beam irradiation without bias correction on mobile plane following the sun at normal incidence (Wh/m2) +# +# Observation period;TOA;Clear sky GHI;Clear sky BHI;Clear sky DHI;Clear sky BNI;GHI;BHI;DHI;BNI;Reliability;sza;summer/winter split;tco3;tcwv;AOD BC;AOD DU;AOD SS;AOD OR;AOD SU;AOD NI;AOD AM;alpha;Aerosol type;fiso;fvol;fgeo;albedo;Cloud optical depth;Cloud coverage;Cloud type;GHI no corr;BHI no corr;DHI no corr;BNI no corr +2020-06-01T12:00:00.0/2020-06-01T12:01:00.0;18.0699;14.1417;12.5594;1.5823;15.3380;13.5893;11.7057;1.8837;14.2954;1.0000;35.0308;0.9723;341.0221;17.7962;0.0065;0.0067;0.0008;0.0215;0.0252;0.0087;0.0022;nan;-1;0.1668;0.0912;0.0267;0.1359;0.000000;0;5;14.1417;12.5594;1.5823;15.3380 +2020-06-01T12:01:00.0/2020-06-01T12:02:00.0;18.0584;14.1311;12.5484;1.5827;15.3343;13.5801;11.6955;1.8846;14.2920;1.0000;35.0828;0.9723;341.0223;17.8020;0.0065;0.0067;0.0008;0.0215;0.0253;0.0087;0.0022;nan;-1;0.1668;0.0912;0.0267;0.1359;0.000000;0;5;14.1311;12.5484;1.5827;15.3343 +2020-06-01T12:02:00.0/2020-06-01T12:03:00.0;18.0467;14.1204;12.5372;1.5831;15.3306;13.5697;11.6849;1.8848;14.2883;1.0000;35.1357;0.9723;341.0224;17.8079;0.0065;0.0067;0.0008;0.0216;0.0253;0.0087;0.0022;nan;-1;0.1668;0.0912;0.0267;0.1359;0.000000;0;5;14.1204;12.5372;1.5831;15.3306 +2020-06-01T12:03:00.0/2020-06-01T12:04:00.0;18.0348;14.1094;12.5259;1.5835;15.3269;13.5602;11.6744;1.8858;14.2849;1.0000;35.1896;0.9723;341.0226;17.8137;0.0065;0.0067;0.0008;0.0217;0.0253;0.0087;0.0022;nan;-1;0.1668;0.0912;0.0267;0.1359;0.000000;0;5;14.1094;12.5259;1.5835;15.3269 diff --git a/pvlib/data/cams_radiation_monthly.csv b/pvlib/data/cams_radiation_monthly.csv new file mode 100644 index 0000000000..0c9b9ba0b7 --- /dev/null +++ b/pvlib/data/cams_radiation_monthly.csv @@ -0,0 +1,47 @@ +# Coding: utf-8 +# File format version: 2 +# Title: CAMS Radiation Service v3.2 all-sky irradiation (derived from satellite data). +# Content: A time-series of solar radiation received on horizontal plane and plane always normal to the sun rays at ground level. +# Returns the global, beam and diffuse irradiations integrated over a selected time step, +# for a selected location (Meteosat Second Generation satellite coverage) and a selected period. +# The research leading to these results has received funding from the European Union within the Copernicus programme. +# Provider: MINES ParisTech (France) +# More information at: http://www.soda-pro.com/web-services/radiation/cams-radiation-service +# Date begin (ISO 8601): 2020-01-01T00:00:00.0 +# Date end (ISO 8601): 2021-02-01T00:00:00.0 +# Latitude (positive North, ISO 19115): 55.7906 +# Longitude (positive East, ISO 19115): 12.5251 +# Altitude (m): 39.00 +# Time reference: Universal time (UT) +# +# Encoding partly from D2.8.III.13-14 INSPIRE Data Specification on Atmospheric Conditions and Meteorological Geographical Features - Technical Guidelines (2013-12-10) and CF (Climate and Forecast) metadata (2013-11-11) +# CF Standard Names registry of ObservablePropertyValue +# http://cfconventions.org/Data/cf-standard-names/27/build/cf-standard-name-table.html +# urn:x-inspire:specification:DS-AC-MF:observable-property-name:cf-standard-name:1.6 +# ObservableProperty +# basePhenomenon:"integral_of_surface_downwelling_shortwave_flux_in_air_sky_wrt_time" +# uom:"Wh m-2" [unit] +# StatisticalMeasure +# statisticalFunction: "sum" +# Summarization (integration) period: 0 year 1 month 0 day 0 h 0 min 0 s +# noValue: nan +# File generated on: 2021-03-07 +# +# Columns: +# 1. Observation period (ISO 8601) +# 2. TOA. Irradiation on horizontal plane at the top of atmosphere (Wh/m2) +# 3. Clear sky GHI. Clear sky global irradiation on horizontal plane at ground level (Wh/m2) +# 4. Clear sky BHI. Clear sky beam irradiation on horizontal plane at ground level (Wh/m2) +# 5. Clear sky DHI. Clear sky diffuse irradiation on horizontal plane at ground level (Wh/m2) +# 6. Clear sky BNI. Clear sky beam irradiation on mobile plane following the sun at normal incidence (Wh/m2) +# 7. GHI. Global irradiation on horizontal plane at ground level (Wh/m2) +# 8. BHI. Beam irradiation on horizontal plane at ground level (Wh/m2) +# 9. DHI. Diffuse irradiation on horizontal plane at ground level (Wh/m2) +#10. BNI. Beam irradiation on mobile plane following the sun at normal incidence (Wh/m2) +#11. Reliability. Proportion of reliable data in the summarization (0-1) +# +# Observation period;TOA;Clear sky GHI;Clear sky BHI;Clear sky DHI;Clear sky BNI;GHI;BHI;DHI;BNI;Reliability +2020-01-01T00:00:00.0/2020-02-01T00:00:00.0;50169.1914;29424.8965;19492.7637;9932.1328;105764.5547;15531.9336;2568.7678;12972.1865;12469.0654;0.9970 +2020-02-01T00:00:00.0/2020-03-01T00:00:00.0;91338.7188;59010.4766;40636.4961;18373.9805;140931.0156;33073.7422;9732.8369;23351.6523;33306.4258;0.9956 +2020-03-01T00:00:00.0/2020-04-01T00:00:00.0;172855.4062;121403.0703;93124.7656;28278.3086;228799.0156;89403.4375;51798.5312;37620.5898;118724.5781;0.9949 +2020-04-01T00:00:00.0/2020-05-01T00:00:00.0;248215.1562;180546.2812;142470.6094;38075.6641;279123.0625;141625.0781;88746.7266;52930.9297;168456.6094;0.9897 diff --git a/pvlib/iotools/cams.py b/pvlib/iotools/cams.py index 7bf3d212e0..2889211d02 100644 --- a/pvlib/iotools/cams.py +++ b/pvlib/iotools/cams.py @@ -119,7 +119,7 @@ def get_cams_radiation(start_date, end_date, latitude, longitude, email, Observation period str Beginning/end of time period TOA, ghi_extra float Horizontal radiation at top of atmosphere Clear sky GHI, ghi_clear float Clear sky global radiation on horizontal - Clear sky BHI, bhi float Clear sky beam radiation on horizontal + Clear sky BHI, bhi_clear float Clear sky beam radiation on horizontal Clear sky DHI, dhi_clear float Clear sky diffuse radiation on horizontal Clear sky BNI, dni_clear float Clear sky beam radiation normal to sun GHI, ghi* float Global horizontal radiation @@ -291,8 +291,6 @@ def parse_cams_radiation(fbuf, integrated=False, label=None, elif (label == 'right') | ((label is None) & (time_step == '1M')): data.index = pd.to_datetime(obs_period.str[1], utc=True) - data.index.name = 'time' # Set index name to None - # Change index for time_step '1d' and '1M' to be date and not datetime if (time_step == '1d') | (time_step == '1M'): data.index = pd.DatetimeIndex(data.index.date) @@ -310,10 +308,12 @@ def parse_cams_radiation(fbuf, integrated=False, label=None, - pd.to_datetime(obs_period.str[0])) hours = time_delta.dt.total_seconds()/60/60 data[integrated_cols] = data[integrated_cols].\ - divide(hours.tolist(), axis='rows') + divide(hours.tolist(), axis='rows').round(4) else: data[integrated_cols] = (data[integrated_cols] / - TIME_STEPS_IN_HOURS[time_step]) + TIME_STEPS_IN_HOURS[time_step]).round(4) + + data.index.name = 'time' # Set index name to None if map_variables: data = data.rename(columns=CAMS_RADIATION_VARIABLE_MAP) diff --git a/pvlib/tests/iotools/test_cams.py b/pvlib/tests/iotools/test_cams.py new file mode 100644 index 0000000000..bd18fe83de --- /dev/null +++ b/pvlib/tests/iotools/test_cams.py @@ -0,0 +1,147 @@ +""" +test iotools for CAMS +""" + +import pandas as pd +import numpy as np +import pytest + +from pvlib.iotools import cams +from conftest import DATA_DIR, assert_frame_equal + + +testfile_mcclear_verbose = DATA_DIR / 'cams_mcclear_1min_verbose.csv' +testfile_mcclear_monthly = DATA_DIR / 'cams_mcclear_monthly.csv' +testfile_radiation_verbose = DATA_DIR / 'cams_radiation_1min_verbose.csv' +testfile_radiation_monthly = DATA_DIR / 'cams_radiation_monthly.csv' + + +index_verbose = pd.date_range('2020-06-01 12', periods=4, freq='1T', tz='UTC') +index_monthly = pd.date_range('2020-01-01', periods=4, freq='1M') + + +dtypes_mcclear_verbose = [ + 'object', 'float64', 'float64', 'float64', 'float64', 'float64', 'float64', + 'float64', 'float64', 'float64', 'float64', 'float64', 'float64', + 'float64', 'float64', 'float64', 'float64', 'float64', 'int64', 'float64', + 'float64', 'float64', 'float64'] + +dtypes_mcclear = [ + 'object', 'float64', 'float64', 'float64', 'float64', 'float64'] + +dtypes_radiation_verbose = [ + 'object', 'float64', 'float64', 'float64', 'float64', 'float64', 'float64', + 'float64', 'float64', 'float64', 'float64', 'float64', 'float64', + 'float64', 'float64', 'float64', 'float64', 'float64', 'float64', + 'float64', 'float64', 'float64', 'float64', 'int64', 'float64', 'float64', + 'float64', 'float64', 'float64', 'int64', 'int64', 'float64', 'float64', + 'float64', 'float64'] + +dtypes_radiation = [ + 'object', 'float64', 'float64', 'float64', 'float64', 'float64', 'float64', + 'float64', 'float64', 'float64', 'float64'] + + +columns_mcclear_verbose = [ + 'Observation period', 'ghi_extra', 'ghi_clear', 'bhi_clear', + 'dhi_clear', 'dni_clear', 'solar_zenith', 'summer/winter split', 'tco3', + 'tcwv', 'AOD BC', 'AOD DU', 'AOD SS', 'AOD OR', 'AOD SU', 'AOD NI', + 'AOD AM', 'alpha', 'Aerosol type', 'fiso', 'fvol', 'fgeo', 'albedo'] + +columns_mcclear = [ + 'Observation period', 'ghi_extra', 'ghi_clear', 'bhi_clear', 'dhi_clear', + 'dni_clear'] + +columns_radiation_verbose = [ + 'Observation period', 'ghi_extra', 'ghi_clear', 'bhi_clear', 'dhi_clear', + 'dni_clear', 'ghi', 'bhi', 'dhi', 'dni', 'Reliability', 'solar_zenith', + 'summer/winter split', 'tco3', 'tcwv', 'AOD BC', 'AOD DU', 'AOD SS', + 'AOD OR', 'AOD SU', 'AOD NI', 'AOD AM', 'alpha', 'Aerosol type', 'fiso', + 'fvol', 'fgeo', 'albedo', 'Cloud optical depth', 'Cloud coverage', + 'Cloud type', 'GHI no corr', 'BHI no corr', 'DHI no corr', 'BNI no corr'] + +columns_radiation = [ + 'Observation period', 'ghi_extra', 'ghi_clear', 'bhi_clear', 'dhi_clear', + 'dni_clear', 'ghi', 'bhi', 'dhi', 'dni', 'Reliability'] + + +values_mcclear_verbose = np.array([ + ['2020-06-01T12:00:00.0/2020-06-01T12:01:00.0', 1084.194, 848.5020, + 753.564, 94.938, 920.28, 35.0308, 0.9723, 341.0221, 17.7962, 0.0065, + 0.0067, 0.0008, 0.0215, 0.0252, 0.0087, 0.0022, np.nan, -1, 0.1668, + 0.0912, 0.0267, 0.1359], + ['2020-06-01T12:01:00.0/2020-06-01T12:02:00.0', 1083.504, 847.866, 752.904, + 94.962, 920.058, 35.0828, 0.9723, 341.0223, 17.802, 0.0065, 0.0067, + 0.0008, 0.0215, 0.0253, 0.0087, 0.0022, np.nan, -1, 0.1668, 0.0912, + 0.0267, 0.1359], + ['2020-06-01T12:02:00.0/2020-06-01T12:03:00.0', 1082.802, 847.224, 752.232, + 94.986, 919.836, 35.1357, 0.9723, 341.0224, 17.8079, 0.0065, 0.0067, + 0.0008, 0.0216, 0.0253, 0.0087, 0.0022, np.nan, -1, 0.1668, 0.0912, + 0.0267, 0.1359], + ['2020-06-01T12:03:00.0/2020-06-01T12:04:00.0', 1082.088, 846.564, 751.554, + 95.01, 919.614, 35.1896, 0.9723, 341.0226, 17.8137, 0.0065, 0.0067, + 0.0008, 0.0217, 0.0253, 0.0087, 0.0022, np.nan, -1, 0.1668, 0.0912, + 0.0267, 0.1359]]) + +values_mcclear_monthly = np.array([ + ['2020-01-01T00:00:00.0/2020-02-01T00:00:00.0', 67.4314, 39.5494, + 26.1998, 13.3496, 142.1562], + ['2020-02-01T00:00:00.0/2020-03-01T00:00:00.0', 131.2335, 84.7849, + 58.3855, 26.3994, 202.4865], + ['2020-03-01T00:00:00.0/2020-04-01T00:00:00.0', 232.3323, 163.176, + 125.1675, 38.0085, 307.5254], + ['2020-04-01T00:00:00.0/2020-05-01T00:00:00.0', 344.7431, 250.7585, + 197.8757, 52.8829, 387.6707]]) + +values_radiation_verbose = np.array([ + ['2020-06-01T12:00:00.0/2020-06-01T12:01:00.0', 1084.194, 848.502, 753.564, + 94.938, 920.28, 815.358, 702.342, 113.022, 857.724, 1.0, 35.0308, 0.9723, + 341.0221, 17.7962, 0.0065, 0.0067, 0.0008, 0.0215, 0.0252, 0.0087, 0.0022, + np.nan, -1, 0.1668, 0.0912, 0.0267, 0.1359, 0.0, 0, 5, 848.502, 753.564, + 94.938, 920.28], + ['2020-06-01T12:01:00.0/2020-06-01T12:02:00.0', 1083.504, 847.866, 752.904, + 94.962, 920.058, 814.806, 701.73, 113.076, 857.52, 1.0, 35.0828, 0.9723, + 341.0223, 17.802, 0.0065, 0.0067, 0.0008, 0.0215, 0.0253, 0.0087, 0.0022, + np.nan, -1, 0.1668, 0.0912, 0.0267, 0.1359, 0.0, 0, 5, 847.866, 752.904, + 94.962, 920.058], + ['2020-06-01T12:02:00.0/2020-06-01T12:03:00.0', 1082.802, 847.224, 752.232, + 94.986, 919.836, 814.182, 701.094, 113.088, 857.298, 1.0, 35.1357, 0.9723, + 341.0224, 17.8079, 0.0065, 0.0067, 0.0008, 0.0216, 0.0253, 0.0087, 0.0022, + np.nan, -1, 0.1668, 0.0912, 0.0267, 0.1359, 0.0, 0, 5, 847.224, 752.232, + 94.986, 919.836], + ['2020-06-01T12:03:00.0/2020-06-01T12:04:00.0', 1082.088, 846.564, 751.554, + 95.01, 919.614, 813.612, 700.464, 113.148, 857.094, 1.0, 35.1896, 0.9723, + 341.0226, 17.8137, 0.0065, 0.0067, 0.0008, 0.0217, 0.0253, 0.0087, 0.0022, + np.nan, -1, 0.1668, 0.0912, 0.0267, 0.1359, 0.0, 0, 5, 846.564, 751.554, + 95.01, 919.614]]) + +values_radiation_monthly = np.array([ + ['2020-01-01T00:00:00.0/2020-02-01T00:00:00.0', 67.4317, 39.5496, + 26.2, 13.3496, 142.1567, 20.8763, 3.4526, 17.4357, 16.7595, 0.997], + ['2020-02-01T00:00:00.0/2020-03-01T00:00:00.0', 131.2338, 84.7852, + 58.3858, 26.3994, 202.4871, 47.5197, 13.984, 33.5512, 47.8541, 0.9956], + ['2020-03-01T00:00:00.0/2020-04-01T00:00:00.0', 232.3325, 163.1762, + 125.1677, 38.0085, 307.5256, 120.1659, 69.6217, 50.5653, 159.576, 0.9949], + ['2020-04-01T00:00:00.0/2020-05-01T00:00:00.0', 344.7433, 250.7587, + 197.8758, 52.8829, 387.6709, 196.7015, 123.2593, 73.5152, 233.9675, + 0.9897]]) + + +@pytest.mark.parametrize('testfile,index,columns,values,dtypes', [ + (testfile_mcclear_verbose, index_verbose, columns_mcclear_verbose, + values_mcclear_verbose, dtypes_mcclear_verbose), + (testfile_mcclear_monthly, index_monthly, columns_mcclear, + values_mcclear_monthly, dtypes_mcclear), + (testfile_radiation_verbose, index_verbose, columns_radiation_verbose, + values_radiation_verbose, dtypes_radiation_verbose), + (testfile_radiation_monthly, index_monthly, columns_radiation, + values_radiation_monthly, dtypes_radiation)]) +def test_read_cams(testfile, index, columns, values, dtypes): + expected = pd.DataFrame(values, columns=columns, index=index) + expected.index.name = 'time' + expected.index.freq = None + for (col, _dtype) in zip(expected.columns, dtypes): + expected[col] = expected[col].astype(_dtype) + out = cams.read_cams_radiation(testfile, integrated=False, + map_variables=True) + assert_frame_equal(out, expected) From 1df6830664fc5dad4876449e2c832cb596f12474 Mon Sep 17 00:00:00 2001 From: AdamRJensen Date: Sun, 7 Mar 2021 19:48:18 +0100 Subject: [PATCH 27/57] Add tests coverage for metadata --- pvlib/iotools/cams.py | 2 +- pvlib/tests/iotools/test_cams.py | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/pvlib/iotools/cams.py b/pvlib/iotools/cams.py index 2889211d02..f66c8741bd 100644 --- a/pvlib/iotools/cams.py +++ b/pvlib/iotools/cams.py @@ -272,7 +272,7 @@ def parse_cams_radiation(fbuf, integrated=False, label=None, if k_new in ['latitude', 'longitude', 'altitude']: meta[k_new] = float(meta.pop(k_old)) - meta['radiation_unit'] = {True: 'Wh/m2', False: 'W/m2'}[integrated] + meta['radiation_unit'] = {True: 'Wh/m^2', False: 'W/m^2'}[integrated] # Determine the time_step from the meta-data dictionary time_step = SUMMATION_PERIOD_TO_TIME_STEP[ diff --git a/pvlib/tests/iotools/test_cams.py b/pvlib/tests/iotools/test_cams.py index bd18fe83de..eedf5b2267 100644 --- a/pvlib/tests/iotools/test_cams.py +++ b/pvlib/tests/iotools/test_cams.py @@ -145,3 +145,15 @@ def test_read_cams(testfile, index, columns, values, dtypes): out = cams.read_cams_radiation(testfile, integrated=False, map_variables=True) assert_frame_equal(out, expected) + + +def test_read_cams_metadata(): + _, meta = cams.read_cams_radiation(testfile_mcclear_monthly, + integrated=False) + assert meta['Time reference'] == 'Universal time (UT)' + assert meta['noValue'] == 'nan' + assert meta['latitude'] == 55.7906 + assert meta['longitude'] == 12.5251 + assert meta['altitude'] == 39.0 + assert meta['radiation_unit'] == 'W/m^2' + assert meta['time_step'] == '1M' From f51824690328194384619ab010eb495b5e3f0bc3 Mon Sep 17 00:00:00 2001 From: "Adam R. Jensen" <39184289+AdamRJensen@users.noreply.github.com> Date: Sun, 7 Mar 2021 22:22:51 +0100 Subject: [PATCH 28/57] Update docs/sphinx/source/whatsnew/v0.9.0.rst Co-authored-by: Will Holmgren --- docs/sphinx/source/whatsnew/v0.9.0.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sphinx/source/whatsnew/v0.9.0.rst b/docs/sphinx/source/whatsnew/v0.9.0.rst index 5038a36ff2..24b46597b5 100644 --- a/docs/sphinx/source/whatsnew/v0.9.0.rst +++ b/docs/sphinx/source/whatsnew/v0.9.0.rst @@ -64,7 +64,7 @@ Enhancements ~~~~~~~~~~~~ * Add :func:`~pvlib.iotools.read_bsrn` for reading BSRN solar radiation data files. (:pull:`1145`, :issue:`1015`) -* Add :func:`~pvlib.iotools.get_cams_radiatoin` and +* Add :func:`~pvlib.iotools.get_cams_radiation` and :func:`~pvlib.iotools.read_cams_radiation` for retrieving and reading CAMS Radiation and McClear time-series files. (:pull:`1175`) From 96e3719ced27853b9eaeaad0790f997cd774dcbb Mon Sep 17 00:00:00 2001 From: AdamRJensen Date: Sun, 7 Mar 2021 22:35:18 +0100 Subject: [PATCH 29/57] Add reference to parsing function in whatsnew --- docs/sphinx/source/whatsnew/v0.9.0.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/sphinx/source/whatsnew/v0.9.0.rst b/docs/sphinx/source/whatsnew/v0.9.0.rst index 24b46597b5..c2e7da88c4 100644 --- a/docs/sphinx/source/whatsnew/v0.9.0.rst +++ b/docs/sphinx/source/whatsnew/v0.9.0.rst @@ -64,9 +64,10 @@ Enhancements ~~~~~~~~~~~~ * Add :func:`~pvlib.iotools.read_bsrn` for reading BSRN solar radiation data files. (:pull:`1145`, :issue:`1015`) -* Add :func:`~pvlib.iotools.get_cams_radiation` and +* Add :func:`~pvlib.iotools.get_cams_radiation`, + :func:`~pvlib.iotools.parse_cams_radiation`, and :func:`~pvlib.iotools.read_cams_radiation` - for retrieving and reading CAMS Radiation and McClear time-series + for retrieving, parsing, and reading CAMS Radiation and McClear time-series files. (:pull:`1175`) * In :py:class:`~pvlib.modelchain.ModelChain`, attributes which contain output of models are now collected into ``ModelChain.results``. From 5f952c0602d6ea0ffbb9812453ff5a2eaa18ab23 Mon Sep 17 00:00:00 2001 From: AdamRJensen Date: Mon, 31 May 2021 19:28:55 +0200 Subject: [PATCH 30/57] Update module and function names Module name changed to sodapro instead of cams. Function names changed, e.g., get_cams instead of get_cams_radiation. --- docs/sphinx/source/api.rst | 6 +-- docs/sphinx/source/whatsnew/v0.9.0.rst | 6 +-- pvlib/iotools/{cams.py => sodapro.py} | 56 ++++++++++++-------------- pvlib/tests/iotools/test_cams.py | 8 ++-- 4 files changed, 35 insertions(+), 41 deletions(-) rename pvlib/iotools/{cams.py => sodapro.py} (89%) diff --git a/docs/sphinx/source/api.rst b/docs/sphinx/source/api.rst index 49ded6f1fa..acd0a6856f 100644 --- a/docs/sphinx/source/api.rst +++ b/docs/sphinx/source/api.rst @@ -484,9 +484,9 @@ relevant to solar energy modeling. iotools.get_pvgis_tmy iotools.read_pvgis_tmy iotools.read_bsrn - iotools.get_cams_radiation - iotools.read_cams_radiation - iotools.parse_cams_radiation + iotools.get_cams + iotools.read_cams + iotools.parse_cams A :py:class:`~pvlib.location.Location` object may be created from metadata in some files. diff --git a/docs/sphinx/source/whatsnew/v0.9.0.rst b/docs/sphinx/source/whatsnew/v0.9.0.rst index c2e7da88c4..0895b2d476 100644 --- a/docs/sphinx/source/whatsnew/v0.9.0.rst +++ b/docs/sphinx/source/whatsnew/v0.9.0.rst @@ -64,9 +64,9 @@ Enhancements ~~~~~~~~~~~~ * Add :func:`~pvlib.iotools.read_bsrn` for reading BSRN solar radiation data files. (:pull:`1145`, :issue:`1015`) -* Add :func:`~pvlib.iotools.get_cams_radiation`, - :func:`~pvlib.iotools.parse_cams_radiation`, and - :func:`~pvlib.iotools.read_cams_radiation` +* Add :func:`~pvlib.iotools.get_cams`, + :func:`~pvlib.iotools.parse_cams`, and + :func:`~pvlib.iotools.read_cams` for retrieving, parsing, and reading CAMS Radiation and McClear time-series files. (:pull:`1175`) * In :py:class:`~pvlib.modelchain.ModelChain`, attributes which contain diff --git a/pvlib/iotools/cams.py b/pvlib/iotools/sodapro.py similarity index 89% rename from pvlib/iotools/cams.py rename to pvlib/iotools/sodapro.py index f66c8741bd..f80b2e3766 100644 --- a/pvlib/iotools/cams.py +++ b/pvlib/iotools/sodapro.py @@ -9,13 +9,13 @@ import warnings -CAMS_RADIATION_INTEGRATED_COLUMNS = [ +CAMS_INTEGRATED_COLUMNS = [ 'TOA', 'Clear sky GHI', 'Clear sky BHI', 'Clear sky DHI', 'Clear sky BNI', 'GHI', 'BHI', 'DHI', 'BNI', 'GHI no corr', 'BHI no corr', 'DHI no corr', 'BNI no corr'] # Dictionary mapping CAMS Radiation and McClear variables to pvlib names -CAMS_RADIATION_VARIABLE_MAP = { +CAMS_VARIABLE_MAP = { 'TOA': 'ghi_extra', 'Clear sky GHI': 'ghi_clear', 'Clear sky BHI': 'bhi_clear', @@ -41,11 +41,10 @@ '0 year 1 month 0 day 0 h 0 min 0 s': '1M'} -def get_cams_radiation(start_date, end_date, latitude, longitude, email, - identifier='mcclear', altitude=None, time_step='1h', - time_ref='UT', verbose=False, integrated=False, - label=None, map_variables=True, - server='www.soda-is.com'): +def get_cams(start_date, end_date, latitude, longitude, email, + identifier='mcclear', altitude=None, time_step='1h', + time_ref='UT', verbose=False, integrated=False, label=None, + map_variables=True, server='www.soda-is.com'): """ Retrieve time-series of radiation and/or clear-sky global, beam, and diffuse radiation from CAMS [1]_, [2]_ using the WGET service [3]_. @@ -70,13 +69,13 @@ def get_cams_radiation(start_date, end_date, latitude, longitude, email, in decimal degrees, between -90 and 90, north is positive (ISO 19115) longitude : float in decimal degrees, between -180 and 180, east is positive (ISO 19115) - altitude: float, default: None - Altitude in meters. If None, then the altitude is determined from the - NASA SRTM database email: str Email address linked to a SoDa account identifier: {'mcclear', 'cams_radiation'} Specify whether to retrieve CAMS Radiation or McClear parameters + altitude: float, default: None + Altitude in meters. If None, then the altitude is determined from the + NASA SRTM database time_step: str, {'1min', '15min', '1h', '1d', '1M'}, default: '1h' Time step of the time series, either 1 minute, 15 minute, hourly, daily, or monthly. @@ -93,7 +92,7 @@ def get_cams_radiation(start_date, end_date, latitude, longitude, email, all time steps except for ‘1M’ which has a default of ‘right’. map_variables: bool, default: True When true, renames columns of the DataFrame to pvlib variable names - where applicable. See variable CAMS_RADIATION_VARIABLE_MAP. + where applicable. See variable CAMS_VARIABLE_MAP. server: str, default: 'www.soda-is.com' Main server (www.soda-is.com) or backup mirror server (pro.soda-is.com) @@ -137,12 +136,12 @@ def get_cams_radiation(start_date, end_date, latitude, longitude, email, Variables corresponding to standard pvlib variables are renamed, e.g. `sza` becomes `solar_zenith`. See the - `pvlib.iotools.cams.CAMS_RADIATION_VARIABLE_MAP` dict for the complete + `pvlib.iotools.cams.CAMS_VARIABLE_MAP` dict for the complete mapping. See Also -------- - pvlib.iotools.read_cams_radiation, pvlib.iotools.parse_cams_radiation + pvlib.iotools.read_cams, pvlib.iotools.parse_cams Raises ------ @@ -211,17 +210,15 @@ def get_cams_radiation(start_date, end_date, latitude, longitude, email, # Successful requests returns a csv data file elif res.headers['Content-Type'] == 'application/csv': fbuf = io.StringIO(res.content.decode('utf-8')) - data, meta = parse_cams_radiation(fbuf, integrated=integrated, - label=label, - map_variables=map_variables) + data, meta = parse_cams(fbuf, integrated=integrated, label=label, + map_variables=map_variables) return data, meta -def parse_cams_radiation(fbuf, integrated=False, label=None, - map_variables=True): +def parse_cams(fbuf, integrated=False, label=None, map_variables=True): """ Parse a file-like buffer with data in the format of a CAMS Radiation or - McClear file. The CAMS servicess are described in [1]_ and [2]_. + McClear file. The CAMS services are described in [1]_ and [2]_. Parameters ---------- @@ -235,7 +232,7 @@ def parse_cams_radiation(fbuf, integrated=False, label=None, all time steps except for ‘1M’ which has a default of ‘right’. map_variables: bool, default: True When true, renames columns of the Dataframe to pvlib variable names - where applicable. See variable CAMS_RADIATION_VARIABLE_MAP. + where applicable. See variable CAMS_VARIABLE_MAP. Returns ------- @@ -246,7 +243,7 @@ def parse_cams_radiation(fbuf, integrated=False, label=None, See Also -------- - pvlib.iotools.read_cams_radiation, pvlib.iotools.get_cams_radiation + pvlib.iotools.read_cams, pvlib.iotools.get_cams References ---------- @@ -300,7 +297,7 @@ def parse_cams_radiation(fbuf, integrated=False, label=None, data.index = data.index - pd.Timedelta(days=1) if not integrated: # Convert radiation values from Wh/m2 to W/m2 - integrated_cols = [c for c in CAMS_RADIATION_INTEGRATED_COLUMNS + integrated_cols = [c for c in CAMS_INTEGRATED_COLUMNS if c in data.columns] if time_step == '1M': @@ -316,13 +313,12 @@ def parse_cams_radiation(fbuf, integrated=False, label=None, data.index.name = 'time' # Set index name to None if map_variables: - data = data.rename(columns=CAMS_RADIATION_VARIABLE_MAP) + data = data.rename(columns=CAMS_VARIABLE_MAP) return data, meta -def read_cams_radiation(filename, integrated=False, label=None, - map_variables=True): +def read_cams(filename, integrated=False, label=None, map_variables=True): """ Read a CAMS Radiation or McClear file into a pandas DataFrame. CAMS radiation and McClear are described in [1]_ and [2]_, respectively. @@ -339,19 +335,19 @@ def read_cams_radiation(filename, integrated=False, label=None, all time steps except for ‘1M’ which has a default of ‘right’. map_variables: bool, default: True When true, renames columns of the Dataframe to pvlib variable names - where applicable. See variable CAMS_RADIATION_VARIABLE_MAP. + where applicable. See variable CAMS_VARIABLE_MAP. Returns ------- data: pandas.DataFrame Timeseries data from CAMS Radiation or McClear - :func:`pvlib.iotools.get_cams_radiation` for fields + :func:`pvlib.iotools.get_cams` for fields meta: dict - Metadata avaiable in the file. + Metadata available in the file. See Also -------- - pvlib.iotools.parse_cams_radiation, pvlib.iotools.get_cams_radiation + pvlib.iotools.parse_cams, pvlib.iotools.get_cams References ---------- @@ -361,5 +357,5 @@ def read_cams_radiation(filename, integrated=False, label=None, `_ """ with open(str(filename), 'r') as fbuf: - content = parse_cams_radiation(fbuf) + content = parse_cams(fbuf) return content diff --git a/pvlib/tests/iotools/test_cams.py b/pvlib/tests/iotools/test_cams.py index eedf5b2267..65b0c95d43 100644 --- a/pvlib/tests/iotools/test_cams.py +++ b/pvlib/tests/iotools/test_cams.py @@ -6,7 +6,7 @@ import numpy as np import pytest -from pvlib.iotools import cams +from pvlib.iotools import sodapro from conftest import DATA_DIR, assert_frame_equal @@ -142,14 +142,12 @@ def test_read_cams(testfile, index, columns, values, dtypes): expected.index.freq = None for (col, _dtype) in zip(expected.columns, dtypes): expected[col] = expected[col].astype(_dtype) - out = cams.read_cams_radiation(testfile, integrated=False, - map_variables=True) + out = sodapro.read_cams(testfile, integrated=False, map_variables=True) assert_frame_equal(out, expected) def test_read_cams_metadata(): - _, meta = cams.read_cams_radiation(testfile_mcclear_monthly, - integrated=False) + _, meta = sodapro.read_cams(testfile_mcclear_monthly, integrated=False) assert meta['Time reference'] == 'Universal time (UT)' assert meta['noValue'] == 'nan' assert meta['latitude'] == 55.7906 From b4a686dc6bf91e39a08cdd24aea7df0678b51962 Mon Sep 17 00:00:00 2001 From: AdamRJensen Date: Mon, 31 May 2021 19:36:24 +0200 Subject: [PATCH 31/57] Update iotools init file with new names --- pvlib/iotools/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pvlib/iotools/__init__.py b/pvlib/iotools/__init__.py index 88f49f3a34..b717c801ca 100644 --- a/pvlib/iotools/__init__.py +++ b/pvlib/iotools/__init__.py @@ -14,6 +14,6 @@ from pvlib.iotools.psm3 import parse_psm3 # noqa: F401 from pvlib.iotools.pvgis import get_pvgis_tmy, read_pvgis_tmy # noqa: F401 from pvlib.iotools.bsrn import read_bsrn # noqa: F401 -from pvlib.iotools.cams import get_cams_radiation # noqa: F401 -from pvlib.iotools.cams import read_cams_radiation # noqa: F401 -from pvlib.iotools.cams import parse_cams_radiation # noqa: F401 +from pvlib.iotools.sodapro import get_cams # noqa: F401 +from pvlib.iotools.sodapro import read_cams # noqa: F401 +from pvlib.iotools.sodapro import parse_cams # noqa: F401 From 1ccbe70cfbaf996924194f291379da4267b48a47 Mon Sep 17 00:00:00 2001 From: AdamRJensen Date: Mon, 31 May 2021 19:43:22 +0200 Subject: [PATCH 32/57] update test file name --- pvlib/tests/iotools/{test_cams.py => test_sodapro.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename pvlib/tests/iotools/{test_cams.py => test_sodapro.py} (100%) diff --git a/pvlib/tests/iotools/test_cams.py b/pvlib/tests/iotools/test_sodapro.py similarity index 100% rename from pvlib/tests/iotools/test_cams.py rename to pvlib/tests/iotools/test_sodapro.py From 5c9ef690c32d873067dddd4949343053341f30d7 Mon Sep 17 00:00:00 2001 From: AdamRJensen Date: Fri, 4 Jun 2021 17:07:38 +0200 Subject: [PATCH 33/57] Add ".." to conftest import and update function documentation Replaced fancy quotation marks with simple ones. --- pvlib/iotools/sodapro.py | 21 +++++++++++---------- pvlib/tests/iotools/test_sodapro.py | 2 +- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/pvlib/iotools/sodapro.py b/pvlib/iotools/sodapro.py index f80b2e3766..79870cf7e2 100644 --- a/pvlib/iotools/sodapro.py +++ b/pvlib/iotools/sodapro.py @@ -47,7 +47,8 @@ def get_cams(start_date, end_date, latitude, longitude, email, map_variables=True, server='www.soda-is.com'): """ Retrieve time-series of radiation and/or clear-sky global, beam, and - diffuse radiation from CAMS [1]_, [2]_ using the WGET service [3]_. + diffuse radiation from CAMS. Data from CAMS Radiation [1]_ and CAMS McClear + [2]_ are retrieved using the WGET service [3]_. Time coverage: 2004-01-01 to two days ago @@ -87,9 +88,9 @@ def get_cams(start_date, end_date, latitude, longitude, email, integrated: boolean, default False Whether to return radiation parameters as integrated values (Wh/m^2) or as average irradiance values (W/m^2) (pvlib preferred units) - label: {‘right’, ‘left’}, default: None - Which bin edge label to label time-step with. The default is ‘left’ for - all time steps except for ‘1M’ which has a default of ‘right’. + label: {'right', 'left'}, default: None + Which bin edge label to label time-step with. The default is 'left for + all time steps except for '1M' which has a default of 'right'. map_variables: bool, default: True When true, renames columns of the DataFrame to pvlib variable names where applicable. See variable CAMS_VARIABLE_MAP. @@ -227,9 +228,9 @@ def parse_cams(fbuf, integrated=False, label=None, map_variables=True): integrated: boolean, default False Whether to return radiation parameters as integrated values (Wh/m^2) or as average irradiance values (W/m^2) (pvlib preferred units) - label: {‘right’, ‘left’}, default: None - Which bin edge label to label time-step with. The default is ‘left’ for - all time steps except for ‘1M’ which has a default of ‘right’. + label: {'right', 'left'}, default: None + Which bin edge label to label time-step with. The default is 'left' for + all time steps except for '1M' which has a default of 'right'. map_variables: bool, default: True When true, renames columns of the Dataframe to pvlib variable names where applicable. See variable CAMS_VARIABLE_MAP. @@ -330,9 +331,9 @@ def read_cams(filename, integrated=False, label=None, map_variables=True): integrated: boolean, default False Whether to return radiation parameters as integrated values (Wh/m^2) or as average irradiance values (W/m^2) (pvlib preferred units) - label: {‘right’, ‘left’}, default: None - Which bin edge label to label time-step with. The default is ‘left’ for - all time steps except for ‘1M’ which has a default of ‘right’. + label: {'right', 'left}, default: None + Which bin edge label to label time-step with. The default is 'left' for + all time steps except for '1M' which has a default of 'right'. map_variables: bool, default: True When true, renames columns of the Dataframe to pvlib variable names where applicable. See variable CAMS_VARIABLE_MAP. diff --git a/pvlib/tests/iotools/test_sodapro.py b/pvlib/tests/iotools/test_sodapro.py index 65b0c95d43..890d03120e 100644 --- a/pvlib/tests/iotools/test_sodapro.py +++ b/pvlib/tests/iotools/test_sodapro.py @@ -7,7 +7,7 @@ import pytest from pvlib.iotools import sodapro -from conftest import DATA_DIR, assert_frame_equal +from ..conftest import DATA_DIR, assert_frame_equal testfile_mcclear_verbose = DATA_DIR / 'cams_mcclear_1min_verbose.csv' From bb34a251a1a535b8a5380c0505ea84845b09aa39 Mon Sep 17 00:00:00 2001 From: AdamRJensen Date: Fri, 4 Jun 2021 17:29:54 +0200 Subject: [PATCH 34/57] =?UTF-8?q?Change=20asterisk=20to=20=E2=80=A0=20in?= =?UTF-8?q?=20documentation?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pvlib/iotools/sodapro.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pvlib/iotools/sodapro.py b/pvlib/iotools/sodapro.py index 79870cf7e2..2c45592e02 100644 --- a/pvlib/iotools/sodapro.py +++ b/pvlib/iotools/sodapro.py @@ -122,14 +122,14 @@ def get_cams(start_date, end_date, latitude, longitude, email, Clear sky BHI, bhi_clear float Clear sky beam radiation on horizontal Clear sky DHI, dhi_clear float Clear sky diffuse radiation on horizontal Clear sky BNI, dni_clear float Clear sky beam radiation normal to sun - GHI, ghi* float Global horizontal radiation - BHI, bhi* float Beam (direct) radiation on horizontal - DHI, dhi* float Diffuse horizontal radiation - BNI, dni* float Beam (direct) radiation normal to the sun - Reliability* float Reliable data fraction in summarization + GHI, ghi† float Global horizontal radiation + BHI, bhi† float Beam (direct) radiation on horizontal + DHI, dhi† float Diffuse horizontal radiation + BNI, dni† float Beam (direct) radiation normal to the sun + Reliability† float Reliable data fraction in summarization ======================== ====== ========================================= - *Parameters only returned if identifier='cams_radiation'. For description + †Parameters only returned if identifier='cams_radiation'. For description of additional output parameters in verbose mode, see [1]_ and [2]_. Note that it is recommended to specify the latitude and longitude to at From 71c3a64d7f843dba7a4e78c74f6ee5b2f2103b9c Mon Sep 17 00:00:00 2001 From: AdamRJensen Date: Fri, 4 Jun 2021 19:31:02 +0200 Subject: [PATCH 35/57] Update from master --- pvlib/iotools/sodapro.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pvlib/iotools/sodapro.py b/pvlib/iotools/sodapro.py index 2c45592e02..ca5ed1d6ba 100644 --- a/pvlib/iotools/sodapro.py +++ b/pvlib/iotools/sodapro.py @@ -201,7 +201,7 @@ def get_cams(start_date, end_date, latitude, longitude, email, # semi-colons, which gets incorrectly formatted by the requests function # if passed using the params argument. res = requests.get(base_url + '?DataInputs=' + data_inputs, params=params) - + print(res.url) # Invalid requests returns an XML error message and the HTTP staus code 200 # as if the request was successful. Therefore, errors cannot be handled # automatic (e.g. res.raise_for_status()) and errors are handled manually From e335730ffb89b4ade4fdfb41c66362d9379ebee2 Mon Sep 17 00:00:00 2001 From: AdamRJensen Date: Fri, 4 Jun 2021 23:52:14 +0200 Subject: [PATCH 36/57] Add mock tests --- pvlib/iotools/sodapro.py | 5 +- pvlib/tests/iotools/test_sodapro.py | 77 ++++++++++++++++++++++++++--- 2 files changed, 75 insertions(+), 7 deletions(-) diff --git a/pvlib/iotools/sodapro.py b/pvlib/iotools/sodapro.py index ca5ed1d6ba..589a976395 100644 --- a/pvlib/iotools/sodapro.py +++ b/pvlib/iotools/sodapro.py @@ -80,7 +80,7 @@ def get_cams(start_date, end_date, latitude, longitude, email, time_step: str, {'1min', '15min', '1h', '1d', '1M'}, default: '1h' Time step of the time series, either 1 minute, 15 minute, hourly, daily, or monthly. - time_reference: str, {'UT', 'TST'}, default: 'UT' + time_ref: str, {'UT', 'TST'}, default: 'UT' 'UT' (universal time) or 'TST' (True Solar Time) verbose: boolean, default: False Verbose mode outputs additional parameters (aerosols). Only available @@ -168,6 +168,9 @@ def get_cams(start_date, end_date, latitude, longitude, email, if (verbose is True) & ((time_step != '1min') | (time_ref != 'UT')): verbose = False warnings.warn("Verbose mode only supports 1 min. UT time series!") + + if identifier not in ['mcclear', 'cams_radiation']: + raise ValueError('Identifier must be either mcclear or cams_radiation') # Format verbose variable to the required format: {'true', 'false'} verbose = str(verbose).lower() diff --git a/pvlib/tests/iotools/test_sodapro.py b/pvlib/tests/iotools/test_sodapro.py index 890d03120e..ec127a61a8 100644 --- a/pvlib/tests/iotools/test_sodapro.py +++ b/pvlib/tests/iotools/test_sodapro.py @@ -4,6 +4,7 @@ import pandas as pd import numpy as np +import requests import pytest from pvlib.iotools import sodapro @@ -127,6 +128,19 @@ 0.9897]]) +# @pytest.fixture() # scope=? +def generate_expected_dataframe(values, columns, index, dtypes): + """Create dataframe from arrays of values, columns and index, in order to + use this dataframe to compare to. + """ + expected = pd.DataFrame(values, columns=columns, index=index) + expected.index.name = 'time' + expected.index.freq = None + for (col, _dtype) in zip(expected.columns, dtypes): + expected[col] = expected[col].astype(_dtype) + return expected + + @pytest.mark.parametrize('testfile,index,columns,values,dtypes', [ (testfile_mcclear_verbose, index_verbose, columns_mcclear_verbose, values_mcclear_verbose, dtypes_mcclear_verbose), @@ -137,12 +151,8 @@ (testfile_radiation_monthly, index_monthly, columns_radiation, values_radiation_monthly, dtypes_radiation)]) def test_read_cams(testfile, index, columns, values, dtypes): - expected = pd.DataFrame(values, columns=columns, index=index) - expected.index.name = 'time' - expected.index.freq = None - for (col, _dtype) in zip(expected.columns, dtypes): - expected[col] = expected[col].astype(_dtype) - out = sodapro.read_cams(testfile, integrated=False, map_variables=True) + expected = generate_expected_dataframe(values, columns, index, dtypes) + out, meta = sodapro.read_cams(testfile, integrated=False, map_variables=True) assert_frame_equal(out, expected) @@ -155,3 +165,58 @@ def test_read_cams_metadata(): assert meta['altitude'] == 39.0 assert meta['radiation_unit'] == 'W/m^2' assert meta['time_step'] == '1M' + + +def test_get_cams(requests_mock): + + with open(testfile_mcclear_monthly, 'r') as test_file: + mock_response = test_file.read() + + url_mcclear_monthly = 'http://www.soda-is.com/service/wps?DataInputs=latitude=55.7906;longitude=12.5251;altitude=80;date_begin=2020-01-01;date_end=2020-05-04;time_ref=UT;summarization=P01M;username=arajen%2540byg.dtu.dk;verbose=false&Service=WPS&Request=Execute&Identifier=get_mcclear&version=1.0.0&RawDataOutput=irradiation' # noqa: E501 + + requests_mock.get(url_mcclear_monthly, text=mock_response, + headers={'Content-Type': 'application/csv'}) + + out, meta = sodapro.get_cams( + start_date=pd.Timestamp('2020-01-01'), + end_date=pd.Timestamp('2020-05-04'), + latitude=55.7906, + longitude=12.5251, + altitude=80, + email='arajen@byg.dtu.dk', + time_step='1M', + integrated=False) + + expected = generate_expected_dataframe( + values_mcclear_monthly,columns_mcclear, index_monthly, dtypes_mcclear) + + assert_frame_equal(out, expected) + + +def test_get_cams_bad_request(requests_mock): + """Test that a HTTP error is raised for invalid requests""" + + # Subset of an xml file returned for errornous requests + mock_response_bad = """ + + Failed to execute WPS process [get_mcclear]: + Please, register yourself at www.soda-pro.com + """ + + url_cams_bad_request = 'http://pro.soda-is.com/service/wps?DataInputs=latitude=55.7906;longitude=12.5251;altitude=-999;date_begin=2020-01-01;date_end=2020-05-04;time_ref=TST;summarization=PT01H;username=test%2540test.com;verbose=false&Service=WPS&Request=Execute&Identifier=get_mcclear&version=1.0.0&RawDataOutput=irradiation' # noqa: E501 + + requests_mock.get(url_cams_bad_request, text=mock_response_bad, + headers={'Content-Type': 'application/xml'}) + + with pytest.raises(requests.HTTPError): + assert sodapro.get_cams( + start_date=pd.Timestamp('2020-01-01'), + end_date=pd.Timestamp('2020-05-04'), + latitude=55.7906, + longitude=12.5251, + identifier='mcclear', + time_ref='TST', + verbose=True, + time_step='1h', + email='test@test.com', # fake email + server='pro.soda-is.com') From 5a018ca1f2a374502ba5ec66210d24c936063d12 Mon Sep 17 00:00:00 2001 From: AdamRJensen Date: Fri, 4 Jun 2021 23:58:15 +0200 Subject: [PATCH 37/57] Stickler fixes and add import of requests_mock --- pvlib/iotools/sodapro.py | 2 +- pvlib/tests/iotools/test_sodapro.py | 28 +++++++++++++++------------- 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/pvlib/iotools/sodapro.py b/pvlib/iotools/sodapro.py index 589a976395..607e1bb035 100644 --- a/pvlib/iotools/sodapro.py +++ b/pvlib/iotools/sodapro.py @@ -168,7 +168,7 @@ def get_cams(start_date, end_date, latitude, longitude, email, if (verbose is True) & ((time_step != '1min') | (time_ref != 'UT')): verbose = False warnings.warn("Verbose mode only supports 1 min. UT time series!") - + if identifier not in ['mcclear', 'cams_radiation']: raise ValueError('Identifier must be either mcclear or cams_radiation') diff --git a/pvlib/tests/iotools/test_sodapro.py b/pvlib/tests/iotools/test_sodapro.py index ec127a61a8..3133ed450d 100644 --- a/pvlib/tests/iotools/test_sodapro.py +++ b/pvlib/tests/iotools/test_sodapro.py @@ -6,6 +6,7 @@ import numpy as np import requests import pytest +import requests_mock from pvlib.iotools import sodapro from ..conftest import DATA_DIR, assert_frame_equal @@ -130,7 +131,7 @@ # @pytest.fixture() # scope=? def generate_expected_dataframe(values, columns, index, dtypes): - """Create dataframe from arrays of values, columns and index, in order to + """Create dataframe from arrays of values, columns and index, in order to use this dataframe to compare to. """ expected = pd.DataFrame(values, columns=columns, index=index) @@ -152,7 +153,8 @@ def generate_expected_dataframe(values, columns, index, dtypes): values_radiation_monthly, dtypes_radiation)]) def test_read_cams(testfile, index, columns, values, dtypes): expected = generate_expected_dataframe(values, columns, index, dtypes) - out, meta = sodapro.read_cams(testfile, integrated=False, map_variables=True) + out, meta = sodapro.read_cams(testfile, integrated=False, + map_variables=True) assert_frame_equal(out, expected) @@ -188,7 +190,7 @@ def test_get_cams(requests_mock): integrated=False) expected = generate_expected_dataframe( - values_mcclear_monthly,columns_mcclear, index_monthly, dtypes_mcclear) + values_mcclear_monthly, columns_mcclear, index_monthly, dtypes_mcclear) assert_frame_equal(out, expected) @@ -210,13 +212,13 @@ def test_get_cams_bad_request(requests_mock): with pytest.raises(requests.HTTPError): assert sodapro.get_cams( - start_date=pd.Timestamp('2020-01-01'), - end_date=pd.Timestamp('2020-05-04'), - latitude=55.7906, - longitude=12.5251, - identifier='mcclear', - time_ref='TST', - verbose=True, - time_step='1h', - email='test@test.com', # fake email - server='pro.soda-is.com') + start_date=pd.Timestamp('2020-01-01'), + end_date=pd.Timestamp('2020-05-04'), + latitude=55.7906, + longitude=12.5251, + identifier='mcclear', + time_ref='TST', + verbose=True, + time_step='1h', + email='test@test.com', # fake email + server='pro.soda-is.com') From d0a25710e1376aaa370b8eeff50cf56a1e3d557f Mon Sep 17 00:00:00 2001 From: AdamRJensen Date: Sat, 5 Jun 2021 00:04:02 +0200 Subject: [PATCH 38/57] Remove import request_mock in test_sodapro.py --- pvlib/tests/iotools/test_sodapro.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pvlib/tests/iotools/test_sodapro.py b/pvlib/tests/iotools/test_sodapro.py index 3133ed450d..e1b02e4d15 100644 --- a/pvlib/tests/iotools/test_sodapro.py +++ b/pvlib/tests/iotools/test_sodapro.py @@ -6,7 +6,6 @@ import numpy as np import requests import pytest -import requests_mock from pvlib.iotools import sodapro from ..conftest import DATA_DIR, assert_frame_equal @@ -218,7 +217,7 @@ def test_get_cams_bad_request(requests_mock): longitude=12.5251, identifier='mcclear', time_ref='TST', - verbose=True, + verbose=False, time_step='1h', email='test@test.com', # fake email server='pro.soda-is.com') From e642399ad67d1da95c5c29d0d12365ec6ba3aa2d Mon Sep 17 00:00:00 2001 From: AdamRJensen Date: Sat, 5 Jun 2021 00:34:22 +0200 Subject: [PATCH 39/57] Added requests-mock to setup.py and CI requirements files --- ci/requirements-py36-min.yml | 1 + ci/requirements-py36.yml | 1 + ci/requirements-py37.yml | 1 + ci/requirements-py38.yml | 1 + ci/requirements-py39.yml | 1 + pvlib/iotools/sodapro.py | 2 +- setup.py | 2 +- 7 files changed, 7 insertions(+), 2 deletions(-) diff --git a/ci/requirements-py36-min.yml b/ci/requirements-py36-min.yml index 29f63c1be1..aafe899f19 100644 --- a/ci/requirements-py36-min.yml +++ b/ci/requirements-py36-min.yml @@ -8,6 +8,7 @@ dependencies: - pytest - pytest-cov - pytest-mock + - requests-mock - pytest-timeout - python=3.6 - pytz diff --git a/ci/requirements-py36.yml b/ci/requirements-py36.yml index fb37fe8404..c49455119f 100644 --- a/ci/requirements-py36.yml +++ b/ci/requirements-py36.yml @@ -16,6 +16,7 @@ dependencies: - pytest - pytest-cov - pytest-mock + - requests-mock - pytest-rerunfailures - pytest-remotedata - pytest-timeout diff --git a/ci/requirements-py37.yml b/ci/requirements-py37.yml index 1542bb35d9..3203b004d1 100644 --- a/ci/requirements-py37.yml +++ b/ci/requirements-py37.yml @@ -16,6 +16,7 @@ dependencies: - pytest - pytest-cov - pytest-mock + - requests-mock - pytest-timeout - pytest-rerunfailures - pytest-remotedata diff --git a/ci/requirements-py38.yml b/ci/requirements-py38.yml index 6db508fd53..ca3a968335 100644 --- a/ci/requirements-py38.yml +++ b/ci/requirements-py38.yml @@ -16,6 +16,7 @@ dependencies: - pytest - pytest-cov - pytest-mock + - requests-mock - pytest-timeout - pytest-rerunfailures - pytest-remotedata diff --git a/ci/requirements-py39.yml b/ci/requirements-py39.yml index 35a3fa1952..16c6449158 100644 --- a/ci/requirements-py39.yml +++ b/ci/requirements-py39.yml @@ -16,6 +16,7 @@ dependencies: - pytest - pytest-cov - pytest-mock + - requests-mock - pytest-timeout - pytest-rerunfailures - pytest-remotedata diff --git a/pvlib/iotools/sodapro.py b/pvlib/iotools/sodapro.py index 607e1bb035..b698ed2b6a 100644 --- a/pvlib/iotools/sodapro.py +++ b/pvlib/iotools/sodapro.py @@ -361,5 +361,5 @@ def read_cams(filename, integrated=False, label=None, map_variables=True): `_ """ with open(str(filename), 'r') as fbuf: - content = parse_cams(fbuf) + content = parse_cams(fbuf, integrated, label, map_variables) return content diff --git a/setup.py b/setup.py index a876f0e995..1c06a319ca 100755 --- a/setup.py +++ b/setup.py @@ -48,7 +48,7 @@ if sys.version_info.major == 3 and sys.version_info.minor == 6: INSTALL_REQUIRES.append('dataclasses') -TESTS_REQUIRE = ['nose', 'pytest', 'pytest-cov', 'pytest-mock', +TESTS_REQUIRE = ['nose', 'pytest', 'pytest-cov', 'pytest-mock', 'requests-mock', 'pytest-timeout', 'pytest-rerunfailures', 'pytest-remotedata'] EXTRAS_REQUIRE = { 'optional': ['cython', 'ephem', 'netcdf4', 'nrel-pysam', 'numba', From 9e69d3eaa602200bf64268dee96403d2b79c5241 Mon Sep 17 00:00:00 2001 From: AdamRJensen Date: Sat, 5 Jun 2021 00:53:10 +0200 Subject: [PATCH 40/57] Add requests-mock to ci\azure\posix.yml --- ci/azure/posix.yml | 2 +- setup.py | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/ci/azure/posix.yml b/ci/azure/posix.yml index cbcdf5694d..086f03dd69 100644 --- a/ci/azure/posix.yml +++ b/ci/azure/posix.yml @@ -23,7 +23,7 @@ jobs: versionSpec: '$(python.version)' - script: | - pip install pytest pytest-cov pytest-mock pytest-timeout pytest-azurepipelines pytest-rerunfailures pytest-remotedata + pip install pytest pytest-cov pytest-mock requests-mock pytest-timeout pytest-azurepipelines pytest-rerunfailures pytest-remotedata pip install -e . pytest pvlib --junitxml=junit/test-results.xml --cov=com --cov-report=xml --cov-report=html displayName: 'Test with pytest' diff --git a/setup.py b/setup.py index 1c06a319ca..216dc34a28 100755 --- a/setup.py +++ b/setup.py @@ -48,8 +48,9 @@ if sys.version_info.major == 3 and sys.version_info.minor == 6: INSTALL_REQUIRES.append('dataclasses') -TESTS_REQUIRE = ['nose', 'pytest', 'pytest-cov', 'pytest-mock', 'requests-mock', - 'pytest-timeout', 'pytest-rerunfailures', 'pytest-remotedata'] +TESTS_REQUIRE = ['nose', 'pytest', 'pytest-cov', 'pytest-mock', + 'requests-mock', 'pytest-timeout', 'pytest-rerunfailures', + 'pytest-remotedata'] EXTRAS_REQUIRE = { 'optional': ['cython', 'ephem', 'netcdf4', 'nrel-pysam', 'numba', 'pvfactors', 'siphon', 'statsmodels', 'tables', From e0853737d985e02fde9e2e24041267a7a43f1372 Mon Sep 17 00:00:00 2001 From: AdamRJensen Date: Fri, 4 Jun 2021 18:08:58 -0500 Subject: [PATCH 41/57] Move requests-mock to pip install section --- ci/requirements-py36.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/requirements-py36.yml b/ci/requirements-py36.yml index c49455119f..1e20b84d0c 100644 --- a/ci/requirements-py36.yml +++ b/ci/requirements-py36.yml @@ -16,7 +16,6 @@ dependencies: - pytest - pytest-cov - pytest-mock - - requests-mock - pytest-rerunfailures - pytest-remotedata - pytest-timeout @@ -31,3 +30,4 @@ dependencies: - dataclasses - nrel-pysam>=2.0 - pvfactors==1.4.1 + - requests-mock From 51b6ab94768546ec6decbd6582fdf2d52a0f68ba Mon Sep 17 00:00:00 2001 From: AdamRJensen Date: Fri, 4 Jun 2021 21:00:58 -0500 Subject: [PATCH 42/57] Move requests-mock to pip install section for py36 and py36-min --- ci/requirements-py36-min.yml | 2 +- ci/requirements-py36.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/requirements-py36-min.yml b/ci/requirements-py36-min.yml index aafe899f19..84adcb360d 100644 --- a/ci/requirements-py36-min.yml +++ b/ci/requirements-py36-min.yml @@ -8,7 +8,6 @@ dependencies: - pytest - pytest-cov - pytest-mock - - requests-mock - pytest-timeout - python=3.6 - pytz @@ -20,3 +19,4 @@ dependencies: - scipy==1.2.0 - pytest-rerunfailures # conda version is >3.6 - pytest-remotedata # conda package is 0.3.0, needs > 0.3.1 + - requests-mock diff --git a/ci/requirements-py36.yml b/ci/requirements-py36.yml index 1e20b84d0c..c49455119f 100644 --- a/ci/requirements-py36.yml +++ b/ci/requirements-py36.yml @@ -16,6 +16,7 @@ dependencies: - pytest - pytest-cov - pytest-mock + - requests-mock - pytest-rerunfailures - pytest-remotedata - pytest-timeout @@ -30,4 +31,3 @@ dependencies: - dataclasses - nrel-pysam>=2.0 - pvfactors==1.4.1 - - requests-mock From 9814caea349b8a8ff0fc91ee5f2f0484ebd15179 Mon Sep 17 00:00:00 2001 From: AdamRJensen Date: Sat, 5 Jun 2021 12:30:21 -0500 Subject: [PATCH 43/57] Add additional tests for full coverage --- pvlib/iotools/sodapro.py | 2 +- pvlib/tests/iotools/test_sodapro.py | 73 ++++++++++++++++++++++++----- 2 files changed, 61 insertions(+), 14 deletions(-) diff --git a/pvlib/iotools/sodapro.py b/pvlib/iotools/sodapro.py index b698ed2b6a..963936e23c 100644 --- a/pvlib/iotools/sodapro.py +++ b/pvlib/iotools/sodapro.py @@ -204,7 +204,7 @@ def get_cams(start_date, end_date, latitude, longitude, email, # semi-colons, which gets incorrectly formatted by the requests function # if passed using the params argument. res = requests.get(base_url + '?DataInputs=' + data_inputs, params=params) - print(res.url) + # Invalid requests returns an XML error message and the HTTP staus code 200 # as if the request was successful. Therefore, errors cannot be handled # automatic (e.g. res.raise_for_status()) and errors are handled manually diff --git a/pvlib/tests/iotools/test_sodapro.py b/pvlib/tests/iotools/test_sodapro.py index e1b02e4d15..faa5f6d50c 100644 --- a/pvlib/tests/iotools/test_sodapro.py +++ b/pvlib/tests/iotools/test_sodapro.py @@ -1,5 +1,5 @@ """ -test iotools for CAMS +test iotools for sodapro """ import pandas as pd @@ -168,34 +168,59 @@ def test_read_cams_metadata(): assert meta['time_step'] == '1M' -def test_get_cams(requests_mock): - - with open(testfile_mcclear_monthly, 'r') as test_file: +@pytest.mark.parametrize('testfile,index,columns,values,dtypes,identifier', [ + (testfile_mcclear_monthly, index_monthly, columns_mcclear, + values_mcclear_monthly, dtypes_mcclear, 'mcclear'), + (testfile_radiation_monthly, index_monthly, columns_radiation, + values_radiation_monthly, dtypes_radiation, 'cams_radiation') + ]) +def test_get_cams(requests_mock, testfile, index, columns, values, dtypes, + identifier): + """Test that get_cams generates the correct URI request and that parse_cams + is being called correctly""" + # Open local test file containing McClear mothly data + with open(testfile, 'r') as test_file: mock_response = test_file.read() + # Specify the full URI of a specific example, this ensures that all of the + # inputs are passing on correctly + url_test_cams = f'http://www.soda-is.com/service/wps?DataInputs=latitude=55.7906;longitude=12.5251;altitude=80;date_begin=2020-01-01;date_end=2020-05-04;time_ref=UT;summarization=P01M;username=arajen%2540byg.dtu.dk;verbose=false&Service=WPS&Request=Execute&Identifier=get_{identifier}&version=1.0.0&RawDataOutput=irradiation' # noqa: E501 - url_mcclear_monthly = 'http://www.soda-is.com/service/wps?DataInputs=latitude=55.7906;longitude=12.5251;altitude=80;date_begin=2020-01-01;date_end=2020-05-04;time_ref=UT;summarization=P01M;username=arajen%2540byg.dtu.dk;verbose=false&Service=WPS&Request=Execute&Identifier=get_mcclear&version=1.0.0&RawDataOutput=irradiation' # noqa: E501 - - requests_mock.get(url_mcclear_monthly, text=mock_response, + requests_mock.get(url_test_cams, text=mock_response, complete_qs=False, headers={'Content-Type': 'application/csv'}) - + # Make API call - an error is raised if the request does not match exactly out, meta = sodapro.get_cams( start_date=pd.Timestamp('2020-01-01'), end_date=pd.Timestamp('2020-05-04'), latitude=55.7906, longitude=12.5251, - altitude=80, email='arajen@byg.dtu.dk', + identifier=identifier, + altitude=80, time_step='1M', + verbose=False, integrated=False) - expected = generate_expected_dataframe( - values_mcclear_monthly, columns_mcclear, index_monthly, dtypes_mcclear) + expected = generate_expected_dataframe(values, columns, index, dtypes) assert_frame_equal(out, expected) + # Test if Warning is raised if verbose mode is True and time_step != '1min' + with pytest.warns(UserWarning): + assert sodapro.get_cams( + start_date=pd.Timestamp('2020-01-01'), + end_date=pd.Timestamp('2020-05-04'), + latitude=55.7906, + longitude=12.5251, + email='arajen@byg.dtu.dk', + identifier=identifier, + altitude=80, + time_step='1M', + verbose=True) + def test_get_cams_bad_request(requests_mock): - """Test that a HTTP error is raised for invalid requests""" + """Test that a the correct errors/warnings ares raised for invalid + requests inputs""" # Subset of an xml file returned for errornous requests mock_response_bad = """ @@ -209,15 +234,37 @@ def test_get_cams_bad_request(requests_mock): requests_mock.get(url_cams_bad_request, text=mock_response_bad, headers={'Content-Type': 'application/xml'}) + # Test if HTTPError is raised if incorrect input is given + # in this example the end_date is errornously before the start_date with pytest.raises(requests.HTTPError): assert sodapro.get_cams( start_date=pd.Timestamp('2020-01-01'), end_date=pd.Timestamp('2020-05-04'), latitude=55.7906, longitude=12.5251, + email='test@test.com', # fake email identifier='mcclear', time_ref='TST', verbose=False, time_step='1h', - email='test@test.com', # fake email server='pro.soda-is.com') + # Test if value error is raised if incorrect identifier is specified + with pytest.raises(ValueError): + assert sodapro.get_cams( + start_date=pd.Timestamp('2020-01-01'), + end_date=pd.Timestamp('2020-05-04'), + latitude=55.7906, + longitude=12.5251, + email='test@test.com', + identifier='test') # incorrect identifier + # Test if value error is raised if incorrect time step is specified + with pytest.raises(ValueError): + assert sodapro.get_cams( + start_date=pd.Timestamp('2020-01-01'), + end_date=pd.Timestamp('2020-05-04'), + latitude=55.7906, + longitude=12.5251, + email='test@test.com', + identifier='mcclear', + time_step = 'test') # incorrect time step + From 1d28d1a19b6bfff1bfe6978aae338d98247e8788 Mon Sep 17 00:00:00 2001 From: AdamRJensen Date: Sat, 5 Jun 2021 12:35:17 -0500 Subject: [PATCH 44/57] Update stickler --- pvlib/tests/iotools/test_sodapro.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/pvlib/tests/iotools/test_sodapro.py b/pvlib/tests/iotools/test_sodapro.py index faa5f6d50c..63193f5e1e 100644 --- a/pvlib/tests/iotools/test_sodapro.py +++ b/pvlib/tests/iotools/test_sodapro.py @@ -172,8 +172,7 @@ def test_read_cams_metadata(): (testfile_mcclear_monthly, index_monthly, columns_mcclear, values_mcclear_monthly, dtypes_mcclear, 'mcclear'), (testfile_radiation_monthly, index_monthly, columns_radiation, - values_radiation_monthly, dtypes_radiation, 'cams_radiation') - ]) + values_radiation_monthly, dtypes_radiation, 'cams_radiation')]) def test_get_cams(requests_mock, testfile, index, columns, values, dtypes, identifier): """Test that get_cams generates the correct URI request and that parse_cams @@ -182,7 +181,7 @@ def test_get_cams(requests_mock, testfile, index, columns, values, dtypes, with open(testfile, 'r') as test_file: mock_response = test_file.read() # Specify the full URI of a specific example, this ensures that all of the - # inputs are passing on correctly + # inputs are passing on correctly url_test_cams = f'http://www.soda-is.com/service/wps?DataInputs=latitude=55.7906;longitude=12.5251;altitude=80;date_begin=2020-01-01;date_end=2020-05-04;time_ref=UT;summarization=P01M;username=arajen%2540byg.dtu.dk;verbose=false&Service=WPS&Request=Execute&Identifier=get_{identifier}&version=1.0.0&RawDataOutput=irradiation' # noqa: E501 requests_mock.get(url_test_cams, text=mock_response, complete_qs=False, @@ -266,5 +265,4 @@ def test_get_cams_bad_request(requests_mock): longitude=12.5251, email='test@test.com', identifier='mcclear', - time_step = 'test') # incorrect time step - + time_step='test') # incorrect time step From 1d115dbbd3e850c5b171cbbb3024db04e3a39d67 Mon Sep 17 00:00:00 2001 From: AdamRJensen Date: Sun, 6 Jun 2021 19:01:29 -0500 Subject: [PATCH 45/57] Add timeout option to get_cams --- pvlib/tests/iotools/test_sodapro.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/pvlib/tests/iotools/test_sodapro.py b/pvlib/tests/iotools/test_sodapro.py index 63193f5e1e..0257794078 100644 --- a/pvlib/tests/iotools/test_sodapro.py +++ b/pvlib/tests/iotools/test_sodapro.py @@ -61,6 +61,12 @@ 'fvol', 'fgeo', 'albedo', 'Cloud optical depth', 'Cloud coverage', 'Cloud type', 'GHI no corr', 'BHI no corr', 'DHI no corr', 'BNI no corr'] +columns_radiation_verbose_unmapped = [ + 'Observation period', 'TOA', 'Clear sky GHI', 'Clear sky BHI', + 'Clear sky DHI', 'Clear sky BNI', 'sza', 'summer/winter split', 'tco3', + 'tcwv', 'AOD BC', 'AOD DU', 'AOD SS', 'AOD OR', 'AOD SU', 'AOD NI', + 'AOD AM', 'alpha', 'Aerosol type', 'fiso', 'fvol', 'fgeo', 'albedo'] + columns_radiation = [ 'Observation period', 'ghi_extra', 'ghi_clear', 'bhi_clear', 'dhi_clear', 'dni_clear', 'ghi', 'bhi', 'dhi', 'dni', 'Reliability'] @@ -157,6 +163,16 @@ def test_read_cams(testfile, index, columns, values, dtypes): assert_frame_equal(out, expected) +# def test_read_cams_integrated_rename_label(): +# expected = generate_expected_dataframe( +# values_radiation_verbose, index_monthly, +# columns_radiation_verbose_unmapped, values_mcclear_monthly, +# dtypes_mcclear) +# out, meta = sodapro.read_cams(testfile_radiation_verbose, integrated=True, +# label='right', map_variables=True) +# assert_frame_equal(out, expected) + + def test_read_cams_metadata(): _, meta = sodapro.read_cams(testfile_mcclear_monthly, integrated=False) assert meta['Time reference'] == 'Universal time (UT)' @@ -186,7 +202,7 @@ def test_get_cams(requests_mock, testfile, index, columns, values, dtypes, requests_mock.get(url_test_cams, text=mock_response, complete_qs=False, headers={'Content-Type': 'application/csv'}) - # Make API call - an error is raised if the request does not match exactly + # Make API call - an error is raised if requested URI does not match out, meta = sodapro.get_cams( start_date=pd.Timestamp('2020-01-01'), end_date=pd.Timestamp('2020-05-04'), From 8e53165a5d160790c3becc3f329b36579923f0dc Mon Sep 17 00:00:00 2001 From: AdamRJensen Date: Sun, 6 Jun 2021 19:06:35 -0500 Subject: [PATCH 46/57] Add timeout option to get_cams --- pvlib/iotools/sodapro.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pvlib/iotools/sodapro.py b/pvlib/iotools/sodapro.py index 963936e23c..1c8e782020 100644 --- a/pvlib/iotools/sodapro.py +++ b/pvlib/iotools/sodapro.py @@ -44,7 +44,7 @@ def get_cams(start_date, end_date, latitude, longitude, email, identifier='mcclear', altitude=None, time_step='1h', time_ref='UT', verbose=False, integrated=False, label=None, - map_variables=True, server='www.soda-is.com'): + map_variables=True, server='www.soda-is.com', timeout=30): """ Retrieve time-series of radiation and/or clear-sky global, beam, and diffuse radiation from CAMS. Data from CAMS Radiation [1]_ and CAMS McClear @@ -96,6 +96,8 @@ def get_cams(start_date, end_date, latitude, longitude, email, where applicable. See variable CAMS_VARIABLE_MAP. server: str, default: 'www.soda-is.com' Main server (www.soda-is.com) or backup mirror server (pro.soda-is.com) + timeout : int, default 30 + Time in seconds to wait for server response before timeout Returns ------- @@ -203,7 +205,8 @@ def get_cams(start_date, end_date, latitude, longitude, email, # added to the base URL as it contains sub-parameters seperated by # semi-colons, which gets incorrectly formatted by the requests function # if passed using the params argument. - res = requests.get(base_url + '?DataInputs=' + data_inputs, params=params) + res = requests.get(base_url + '?DataInputs=' + data_inputs, params=params, + timeout=timeout) # Invalid requests returns an XML error message and the HTTP staus code 200 # as if the request was successful. Therefore, errors cannot be handled From 3714e62665c564b6841f468ea8920800db4996f8 Mon Sep 17 00:00:00 2001 From: AdamRJensen Date: Sun, 6 Jun 2021 20:26:32 -0500 Subject: [PATCH 47/57] Extent tests to cover label, integrated, map_variables arguments --- pvlib/tests/iotools/test_sodapro.py | 33 +++++++++++++++++++---------- 1 file changed, 22 insertions(+), 11 deletions(-) diff --git a/pvlib/tests/iotools/test_sodapro.py b/pvlib/tests/iotools/test_sodapro.py index 0257794078..4c9d23350c 100644 --- a/pvlib/tests/iotools/test_sodapro.py +++ b/pvlib/tests/iotools/test_sodapro.py @@ -63,9 +63,12 @@ columns_radiation_verbose_unmapped = [ 'Observation period', 'TOA', 'Clear sky GHI', 'Clear sky BHI', - 'Clear sky DHI', 'Clear sky BNI', 'sza', 'summer/winter split', 'tco3', - 'tcwv', 'AOD BC', 'AOD DU', 'AOD SS', 'AOD OR', 'AOD SU', 'AOD NI', - 'AOD AM', 'alpha', 'Aerosol type', 'fiso', 'fvol', 'fgeo', 'albedo'] + 'Clear sky DHI', 'Clear sky BNI', 'GHI', 'BHI', 'DHI', 'BNI', + 'Reliability', 'sza', 'summer/winter split', 'tco3', 'tcwv', 'AOD BC', + 'AOD DU', 'AOD SS', 'AOD OR', 'AOD SU', 'AOD NI', 'AOD AM', 'alpha', + 'Aerosol type', 'fiso', 'fvol', 'fgeo', 'albedo', 'Cloud optical depth', + 'Cloud coverage', 'Cloud type', 'GHI no corr', 'BHI no corr', + 'DHI no corr', 'BNI no corr'] columns_radiation = [ 'Observation period', 'ghi_extra', 'ghi_clear', 'bhi_clear', 'dhi_clear', @@ -122,6 +125,12 @@ np.nan, -1, 0.1668, 0.0912, 0.0267, 0.1359, 0.0, 0, 5, 846.564, 751.554, 95.01, 919.614]]) +values_radiation_verbose_integrated = np.copy(values_radiation_verbose) +values_radiation_verbose_integrated[:,1:10] = \ + values_radiation_verbose_integrated[:,1:10].astype(float)/60 +values_radiation_verbose_integrated[:,31:35] = \ + values_radiation_verbose_integrated[:,31:35].astype(float)/60 + values_radiation_monthly = np.array([ ['2020-01-01T00:00:00.0/2020-02-01T00:00:00.0', 67.4317, 39.5496, 26.2, 13.3496, 142.1567, 20.8763, 3.4526, 17.4357, 16.7595, 0.997], @@ -163,14 +172,16 @@ def test_read_cams(testfile, index, columns, values, dtypes): assert_frame_equal(out, expected) -# def test_read_cams_integrated_rename_label(): -# expected = generate_expected_dataframe( -# values_radiation_verbose, index_monthly, -# columns_radiation_verbose_unmapped, values_mcclear_monthly, -# dtypes_mcclear) -# out, meta = sodapro.read_cams(testfile_radiation_verbose, integrated=True, -# label='right', map_variables=True) -# assert_frame_equal(out, expected) +def test_read_cams_integrated_unmapped_label(): + # Default label is 'left' for 1 minute time resolution, hence 1 minute is + # added for label='right' + expected = generate_expected_dataframe( + values_radiation_verbose_integrated, + columns_radiation_verbose_unmapped, + index_verbose+pd.Timedelta(minutes=1), dtypes=dtypes_radiation_verbose) + out, meta = sodapro.read_cams(testfile_radiation_verbose, integrated=True, + label='right', map_variables=False) + assert_frame_equal(out, expected) def test_read_cams_metadata(): From cc5347ddb6d8ca2d47b23bfa5449214aae543d2e Mon Sep 17 00:00:00 2001 From: AdamRJensen Date: Sun, 6 Jun 2021 20:31:06 -0500 Subject: [PATCH 48/57] Fix stickler --- pvlib/tests/iotools/test_sodapro.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pvlib/tests/iotools/test_sodapro.py b/pvlib/tests/iotools/test_sodapro.py index 4c9d23350c..5153e82eb9 100644 --- a/pvlib/tests/iotools/test_sodapro.py +++ b/pvlib/tests/iotools/test_sodapro.py @@ -126,10 +126,10 @@ 95.01, 919.614]]) values_radiation_verbose_integrated = np.copy(values_radiation_verbose) -values_radiation_verbose_integrated[:,1:10] = \ - values_radiation_verbose_integrated[:,1:10].astype(float)/60 -values_radiation_verbose_integrated[:,31:35] = \ - values_radiation_verbose_integrated[:,31:35].astype(float)/60 +values_radiation_verbose_integrated[:, 1:10] = \ + values_radiation_verbose_integrated[:, 1:10].astype(float)/60 +values_radiation_verbose_integrated[:, 31:35] = \ + values_radiation_verbose_integrated[:, 31:35].astype(float)/60 values_radiation_monthly = np.array([ ['2020-01-01T00:00:00.0/2020-02-01T00:00:00.0', 67.4317, 39.5496, From 9fc945f73f51573854b41a966294e66e8d25aa37 Mon Sep 17 00:00:00 2001 From: "Adam R. Jensen" <39184289+AdamRJensen@users.noreply.github.com> Date: Mon, 7 Jun 2021 12:28:56 -0500 Subject: [PATCH 49/57] Updates to documentation Co-authored-by: Kevin Anderson <57452607+kanderso-nrel@users.noreply.github.com> --- pvlib/iotools/sodapro.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pvlib/iotools/sodapro.py b/pvlib/iotools/sodapro.py index 1c8e782020..7cc205321c 100644 --- a/pvlib/iotools/sodapro.py +++ b/pvlib/iotools/sodapro.py @@ -84,7 +84,7 @@ def get_cams(start_date, end_date, latitude, longitude, email, 'UT' (universal time) or 'TST' (True Solar Time) verbose: boolean, default: False Verbose mode outputs additional parameters (aerosols). Only available - for 1 minute and universal time. See [1] for parameter description. + for 1 minute and universal time. See [1]_ for parameter description. integrated: boolean, default False Whether to return radiation parameters as integrated values (Wh/m^2) or as average irradiance values (W/m^2) (pvlib preferred units) From 8d85f0b5b31c20c5039908857d1807479caa93db Mon Sep 17 00:00:00 2001 From: "Adam R. Jensen" <39184289+AdamRJensen@users.noreply.github.com> Date: Mon, 7 Jun 2021 12:29:20 -0500 Subject: [PATCH 50/57] Update to documentation Co-authored-by: Kevin Anderson <57452607+kanderso-nrel@users.noreply.github.com> --- pvlib/iotools/sodapro.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pvlib/iotools/sodapro.py b/pvlib/iotools/sodapro.py index 7cc205321c..6d9b67e35a 100644 --- a/pvlib/iotools/sodapro.py +++ b/pvlib/iotools/sodapro.py @@ -89,7 +89,7 @@ def get_cams(start_date, end_date, latitude, longitude, email, Whether to return radiation parameters as integrated values (Wh/m^2) or as average irradiance values (W/m^2) (pvlib preferred units) label: {'right', 'left'}, default: None - Which bin edge label to label time-step with. The default is 'left for + Which bin edge label to label time-step with. The default is 'left' for all time steps except for '1M' which has a default of 'right'. map_variables: bool, default: True When true, renames columns of the DataFrame to pvlib variable names From 857bd918e9ed0a5b8e47e5c0e7da325d65c27a48 Mon Sep 17 00:00:00 2001 From: AdamRJensen Date: Mon, 7 Jun 2021 12:44:43 -0500 Subject: [PATCH 51/57] Reformat data inputs, set index name to None Also, includes a few minor changes addressing the review by @kanderso-nrel --- pvlib/iotools/sodapro.py | 30 ++++++++++++++++++----------- pvlib/tests/iotools/test_sodapro.py | 1 - 2 files changed, 19 insertions(+), 12 deletions(-) diff --git a/pvlib/iotools/sodapro.py b/pvlib/iotools/sodapro.py index 6d9b67e35a..04bac953d7 100644 --- a/pvlib/iotools/sodapro.py +++ b/pvlib/iotools/sodapro.py @@ -48,7 +48,7 @@ def get_cams(start_date, end_date, latitude, longitude, email, """ Retrieve time-series of radiation and/or clear-sky global, beam, and diffuse radiation from CAMS. Data from CAMS Radiation [1]_ and CAMS McClear - [2]_ are retrieved using the WGET service [3]_. + [2]_ are retrieved from SoDa [3]_. Time coverage: 2004-01-01 to two days ago @@ -167,7 +167,7 @@ def get_cams(start_date, end_date, latitude, longitude, email, raise ValueError(f'Time step not recognized. Must be one of ' f'{list(TIME_STEPS_MAP.keys())}') - if (verbose is True) & ((time_step != '1min') | (time_ref != 'UT')): + if (verbose) and ((time_step != '1min') or (time_ref != 'UT')): verbose = False warnings.warn("Verbose mode only supports 1 min. UT time series!") @@ -189,10 +189,20 @@ def get_cams(start_date, end_date, latitude, longitude, email, base_url = f"http://{server}/service/wps" - data_inputs = ( - f"latitude={latitude};longitude={longitude};altitude={altitude};" - f"date_begin={start_date};date_end={end_date};time_ref={time_ref};" - f"summarization={time_step_str};username={email};verbose={verbose}") + data_inputs_dict = { + 'latitude': latitude, + 'longitude': longitude, + 'altitude': altitude, + 'date_begin': start_date, + 'date_end': end_date, + 'time_ref': time_ref, + 'summarization': time_step_str, + 'username': email, + 'verbose': verbose} + + # Manual formatting of the input parameters seperating each by a semicolon + data_inputs = ";".join([f"{key}={value}" for key, value in + data_inputs_dict.items()]) params = {'Service': 'WPS', 'Request': 'Execute', @@ -246,7 +256,7 @@ def parse_cams(fbuf, integrated=False, label=None, map_variables=True): data: pandas.DataFrame Timeseries data from CAMS Radiation or McClear meta: dict - Metadata avaiable in the file. + Metadata available in the file. See Also -------- @@ -295,7 +305,7 @@ def parse_cams(fbuf, integrated=False, label=None, map_variables=True): elif (label == 'right') | ((label is None) & (time_step == '1M')): data.index = pd.to_datetime(obs_period.str[1], utc=True) - # Change index for time_step '1d' and '1M' to be date and not datetime + # For time_steps '1d' and '1M', drop timezone and round to nearest midnight if (time_step == '1d') | (time_step == '1M'): data.index = pd.DatetimeIndex(data.index.date) # For monthly data with 'right' label, the index should be the last @@ -316,9 +326,7 @@ def parse_cams(fbuf, integrated=False, label=None, map_variables=True): else: data[integrated_cols] = (data[integrated_cols] / TIME_STEPS_IN_HOURS[time_step]).round(4) - - data.index.name = 'time' # Set index name to None - + data.index.name = None # Set index name to None if map_variables: data = data.rename(columns=CAMS_VARIABLE_MAP) diff --git a/pvlib/tests/iotools/test_sodapro.py b/pvlib/tests/iotools/test_sodapro.py index 5153e82eb9..0863eee9b1 100644 --- a/pvlib/tests/iotools/test_sodapro.py +++ b/pvlib/tests/iotools/test_sodapro.py @@ -149,7 +149,6 @@ def generate_expected_dataframe(values, columns, index, dtypes): use this dataframe to compare to. """ expected = pd.DataFrame(values, columns=columns, index=index) - expected.index.name = 'time' expected.index.freq = None for (col, _dtype) in zip(expected.columns, dtypes): expected[col] = expected[col].astype(_dtype) From 15e097cd96b0961ed544b7e8009d6930d2fad377 Mon Sep 17 00:00:00 2001 From: AdamRJensen Date: Mon, 7 Jun 2021 12:56:38 -0500 Subject: [PATCH 52/57] Remove rounding when converting from integrated values --- pvlib/iotools/sodapro.py | 2 +- pvlib/tests/iotools/test_sodapro.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pvlib/iotools/sodapro.py b/pvlib/iotools/sodapro.py index 04bac953d7..0cad50f420 100644 --- a/pvlib/iotools/sodapro.py +++ b/pvlib/iotools/sodapro.py @@ -322,7 +322,7 @@ def parse_cams(fbuf, integrated=False, label=None, map_variables=True): - pd.to_datetime(obs_period.str[0])) hours = time_delta.dt.total_seconds()/60/60 data[integrated_cols] = data[integrated_cols].\ - divide(hours.tolist(), axis='rows').round(4) + divide(hours.tolist(), axis='rows') else: data[integrated_cols] = (data[integrated_cols] / TIME_STEPS_IN_HOURS[time_step]).round(4) diff --git a/pvlib/tests/iotools/test_sodapro.py b/pvlib/tests/iotools/test_sodapro.py index 0863eee9b1..d2b2ff1131 100644 --- a/pvlib/tests/iotools/test_sodapro.py +++ b/pvlib/tests/iotools/test_sodapro.py @@ -168,7 +168,7 @@ def test_read_cams(testfile, index, columns, values, dtypes): expected = generate_expected_dataframe(values, columns, index, dtypes) out, meta = sodapro.read_cams(testfile, integrated=False, map_variables=True) - assert_frame_equal(out, expected) + assert_frame_equal(out, expected, check_less_precise=True) def test_read_cams_integrated_unmapped_label(): @@ -180,7 +180,7 @@ def test_read_cams_integrated_unmapped_label(): index_verbose+pd.Timedelta(minutes=1), dtypes=dtypes_radiation_verbose) out, meta = sodapro.read_cams(testfile_radiation_verbose, integrated=True, label='right', map_variables=False) - assert_frame_equal(out, expected) + assert_frame_equal(out, expected, check_less_precise=True) def test_read_cams_metadata(): @@ -227,7 +227,7 @@ def test_get_cams(requests_mock, testfile, index, columns, values, dtypes, expected = generate_expected_dataframe(values, columns, index, dtypes) - assert_frame_equal(out, expected) + assert_frame_equal(out, expected, check_less_precise=True) # Test if Warning is raised if verbose mode is True and time_step != '1min' with pytest.warns(UserWarning): From e80cbde2f985c085afa2a48178e17a9037f37220 Mon Sep 17 00:00:00 2001 From: AdamRJensen Date: Mon, 7 Jun 2021 13:40:11 -0500 Subject: [PATCH 53/57] Minor updates to mock tests Includes match keywords for tests that asserts warning and error messages. --- pvlib/tests/iotools/test_sodapro.py | 43 +++++++++++++++-------------- 1 file changed, 22 insertions(+), 21 deletions(-) diff --git a/pvlib/tests/iotools/test_sodapro.py b/pvlib/tests/iotools/test_sodapro.py index d2b2ff1131..0d8ed726bf 100644 --- a/pvlib/tests/iotools/test_sodapro.py +++ b/pvlib/tests/iotools/test_sodapro.py @@ -143,7 +143,7 @@ 0.9897]]) -# @pytest.fixture() # scope=? +# @pytest.fixture def generate_expected_dataframe(values, columns, index, dtypes): """Create dataframe from arrays of values, columns and index, in order to use this dataframe to compare to. @@ -208,35 +208,34 @@ def test_get_cams(requests_mock, testfile, index, columns, values, dtypes, mock_response = test_file.read() # Specify the full URI of a specific example, this ensures that all of the # inputs are passing on correctly - url_test_cams = f'http://www.soda-is.com/service/wps?DataInputs=latitude=55.7906;longitude=12.5251;altitude=80;date_begin=2020-01-01;date_end=2020-05-04;time_ref=UT;summarization=P01M;username=arajen%2540byg.dtu.dk;verbose=false&Service=WPS&Request=Execute&Identifier=get_{identifier}&version=1.0.0&RawDataOutput=irradiation' # noqa: E501 + url_test_cams = f'http://www.soda-is.com/service/wps?DataInputs=latitude=55.7906;longitude=12.5251;altitude=80;date_begin=2020-01-01;date_end=2020-05-04;time_ref=UT;summarization=P01M;username=pvlib-admin%2540googlegroups.com;verbose=false&Service=WPS&Request=Execute&Identifier=get_{identifier}&version=1.0.0&RawDataOutput=irradiation' # noqa: E501 - requests_mock.get(url_test_cams, text=mock_response, complete_qs=False, + requests_mock.get(url_test_cams, text=mock_response, headers={'Content-Type': 'application/csv'}) + # Make API call - an error is raised if requested URI does not match out, meta = sodapro.get_cams( start_date=pd.Timestamp('2020-01-01'), end_date=pd.Timestamp('2020-05-04'), latitude=55.7906, longitude=12.5251, - email='arajen@byg.dtu.dk', + email='pvlib-admin@googlegroups.com', identifier=identifier, altitude=80, time_step='1M', verbose=False, integrated=False) - expected = generate_expected_dataframe(values, columns, index, dtypes) - assert_frame_equal(out, expected, check_less_precise=True) # Test if Warning is raised if verbose mode is True and time_step != '1min' - with pytest.warns(UserWarning): - assert sodapro.get_cams( + with pytest.warns(UserWarning, match='Verbose mode only supports'): + _ = sodapro.get_cams( start_date=pd.Timestamp('2020-01-01'), end_date=pd.Timestamp('2020-05-04'), latitude=55.7906, longitude=12.5251, - email='arajen@byg.dtu.dk', + email='pvlib-admin@googlegroups.com', identifier=identifier, altitude=80, time_step='1M', @@ -245,7 +244,7 @@ def test_get_cams(requests_mock, testfile, index, columns, values, dtypes, def test_get_cams_bad_request(requests_mock): """Test that a the correct errors/warnings ares raised for invalid - requests inputs""" + requests inputs. Also tests if the specified server url gets used""" # Subset of an xml file returned for errornous requests mock_response_bad = """ @@ -259,36 +258,38 @@ def test_get_cams_bad_request(requests_mock): requests_mock.get(url_cams_bad_request, text=mock_response_bad, headers={'Content-Type': 'application/xml'}) - # Test if HTTPError is raised if incorrect input is given - # in this example the end_date is errornously before the start_date - with pytest.raises(requests.HTTPError): - assert sodapro.get_cams( + # Test if HTTPError is raised if incorrect input is specified + # In the below example a non-registrered email is specified + with pytest.raises(requests.HTTPError, match='Failed to execute WPS'): + _ = sodapro.get_cams( start_date=pd.Timestamp('2020-01-01'), end_date=pd.Timestamp('2020-05-04'), latitude=55.7906, longitude=12.5251, - email='test@test.com', # fake email + email='test@test.com', # a non-registrered email identifier='mcclear', time_ref='TST', verbose=False, time_step='1h', server='pro.soda-is.com') # Test if value error is raised if incorrect identifier is specified - with pytest.raises(ValueError): - assert sodapro.get_cams( + with pytest.raises(ValueError, match='Identifier must be either'): + _ = sodapro.get_cams( start_date=pd.Timestamp('2020-01-01'), end_date=pd.Timestamp('2020-05-04'), latitude=55.7906, longitude=12.5251, email='test@test.com', - identifier='test') # incorrect identifier + identifier='test', # incorrect identifier + server='pro.soda-is.com') # Test if value error is raised if incorrect time step is specified - with pytest.raises(ValueError): - assert sodapro.get_cams( + with pytest.raises(ValueError, match='Time step not recognized'): + _ = sodapro.get_cams( start_date=pd.Timestamp('2020-01-01'), end_date=pd.Timestamp('2020-05-04'), latitude=55.7906, longitude=12.5251, email='test@test.com', identifier='mcclear', - time_step='test') # incorrect time step + time_step='test', # incorrect time step + server='pro.soda-is.com') From 6aadd5c8e5e4c67111316ee4ca58853f2c8e14fc Mon Sep 17 00:00:00 2001 From: AdamRJensen Date: Mon, 7 Jun 2021 13:41:18 -0500 Subject: [PATCH 54/57] Remove round(4) --- pvlib/iotools/sodapro.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pvlib/iotools/sodapro.py b/pvlib/iotools/sodapro.py index 0cad50f420..c770a6d17c 100644 --- a/pvlib/iotools/sodapro.py +++ b/pvlib/iotools/sodapro.py @@ -325,7 +325,7 @@ def parse_cams(fbuf, integrated=False, label=None, map_variables=True): divide(hours.tolist(), axis='rows') else: data[integrated_cols] = (data[integrated_cols] / - TIME_STEPS_IN_HOURS[time_step]).round(4) + TIME_STEPS_IN_HOURS[time_step]) data.index.name = None # Set index name to None if map_variables: data = data.rename(columns=CAMS_VARIABLE_MAP) From f194905390a01f757ff642ca469163e5c55a2ac2 Mon Sep 17 00:00:00 2001 From: AdamRJensen Date: Fri, 11 Jun 2021 12:23:58 -0500 Subject: [PATCH 55/57] Change meta variable name to metadata --- pvlib/iotools/sodapro.py | 35 +++++++++++++++-------------- pvlib/tests/iotools/test_sodapro.py | 25 +++++++++++---------- 2 files changed, 31 insertions(+), 29 deletions(-) diff --git a/pvlib/iotools/sodapro.py b/pvlib/iotools/sodapro.py index c770a6d17c..8ea0e12066 100644 --- a/pvlib/iotools/sodapro.py +++ b/pvlib/iotools/sodapro.py @@ -103,7 +103,7 @@ def get_cams(start_date, end_date, latitude, longitude, email, ------- data: pandas.DataFrame Timeseries data, see Notes for columns - meta: dict + metadata: dict Metadata of the requested time-series Notes @@ -227,9 +227,9 @@ def get_cams(start_date, end_date, latitude, longitude, email, # Successful requests returns a csv data file elif res.headers['Content-Type'] == 'application/csv': fbuf = io.StringIO(res.content.decode('utf-8')) - data, meta = parse_cams(fbuf, integrated=integrated, label=label, + data, metadata = parse_cams(fbuf, integrated=integrated, label=label, map_variables=map_variables) - return data, meta + return data, metadata def parse_cams(fbuf, integrated=False, label=None, map_variables=True): @@ -255,7 +255,7 @@ def parse_cams(fbuf, integrated=False, label=None, map_variables=True): ------- data: pandas.DataFrame Timeseries data from CAMS Radiation or McClear - meta: dict + metadata: dict Metadata available in the file. See Also @@ -269,29 +269,30 @@ def parse_cams(fbuf, integrated=False, label=None, map_variables=True): .. [2] `CAMS McClear Service Info `_ """ - meta = {} - # Initial lines starting with # contain meta-data + metadata = {} + # Initial lines starting with # contain metadata while True: line = fbuf.readline().rstrip('\n') if line.startswith('# Observation period'): - # The last line of the meta-data section contains the column names + # The last line of the metadata section contains the column names names = line.lstrip('# ').split(';') - break # End of meta-data section has been reached + break # End of metadata section has been reached elif ': ' in line: - meta[line.split(': ')[0].lstrip('# ')] = line.split(': ')[1] + metadata[line.split(': ')[0].lstrip('# ')] = line.split(': ')[1] # Convert latitude, longitude, and altitude values from strings to floats - for k_old in list(meta.keys()): + for k_old in list(metadata.keys()): k_new = k_old.lstrip().split(' ')[0].lower() if k_new in ['latitude', 'longitude', 'altitude']: - meta[k_new] = float(meta.pop(k_old)) + metadata[k_new] = float(metadata.pop(k_old)) - meta['radiation_unit'] = {True: 'Wh/m^2', False: 'W/m^2'}[integrated] + mmetadataeta['radiation_unit'] = \ + {True: 'Wh/m^2', False: 'W/m^2'}[integrated] - # Determine the time_step from the meta-data dictionary + # Determine the time_step from the metadata dictionary time_step = SUMMATION_PERIOD_TO_TIME_STEP[ - meta['Summarization (integration) period']] - meta['time_step'] = time_step + metadata['Summarization (integration) period']] + mmetadataeta['time_step'] = time_step data = pd.read_csv(fbuf, sep=';', comment='#', header=None, names=names) @@ -330,7 +331,7 @@ def parse_cams(fbuf, integrated=False, label=None, map_variables=True): if map_variables: data = data.rename(columns=CAMS_VARIABLE_MAP) - return data, meta + return data, metadata def read_cams(filename, integrated=False, label=None, map_variables=True): @@ -357,7 +358,7 @@ def read_cams(filename, integrated=False, label=None, map_variables=True): data: pandas.DataFrame Timeseries data from CAMS Radiation or McClear :func:`pvlib.iotools.get_cams` for fields - meta: dict + metadata: dict Metadata available in the file. See Also diff --git a/pvlib/tests/iotools/test_sodapro.py b/pvlib/tests/iotools/test_sodapro.py index 0d8ed726bf..47cebdc01b 100644 --- a/pvlib/tests/iotools/test_sodapro.py +++ b/pvlib/tests/iotools/test_sodapro.py @@ -166,7 +166,7 @@ def generate_expected_dataframe(values, columns, index, dtypes): values_radiation_monthly, dtypes_radiation)]) def test_read_cams(testfile, index, columns, values, dtypes): expected = generate_expected_dataframe(values, columns, index, dtypes) - out, meta = sodapro.read_cams(testfile, integrated=False, + out, metadata = sodapro.read_cams(testfile, integrated=False, map_variables=True) assert_frame_equal(out, expected, check_less_precise=True) @@ -178,20 +178,21 @@ def test_read_cams_integrated_unmapped_label(): values_radiation_verbose_integrated, columns_radiation_verbose_unmapped, index_verbose+pd.Timedelta(minutes=1), dtypes=dtypes_radiation_verbose) - out, meta = sodapro.read_cams(testfile_radiation_verbose, integrated=True, - label='right', map_variables=False) + out, metadata = sodapro.read_cams(testfile_radiation_verbose, + integrated=True, label='right', + map_variables=False) assert_frame_equal(out, expected, check_less_precise=True) def test_read_cams_metadata(): - _, meta = sodapro.read_cams(testfile_mcclear_monthly, integrated=False) - assert meta['Time reference'] == 'Universal time (UT)' - assert meta['noValue'] == 'nan' - assert meta['latitude'] == 55.7906 - assert meta['longitude'] == 12.5251 - assert meta['altitude'] == 39.0 - assert meta['radiation_unit'] == 'W/m^2' - assert meta['time_step'] == '1M' + _, metadata = sodapro.read_cams(testfile_mcclear_monthly, integrated=False) + assert metadata['Time reference'] == 'Universal time (UT)' + assert metadata['noValue'] == 'nan' + assert metadata['latitude'] == 55.7906 + assert metadata['longitude'] == 12.5251 + assert metadata['altitude'] == 39.0 + assert metadata['radiation_unit'] == 'W/m^2' + assert metadata['time_step'] == '1M' @pytest.mark.parametrize('testfile,index,columns,values,dtypes,identifier', [ @@ -214,7 +215,7 @@ def test_get_cams(requests_mock, testfile, index, columns, values, dtypes, headers={'Content-Type': 'application/csv'}) # Make API call - an error is raised if requested URI does not match - out, meta = sodapro.get_cams( + out, metadata = sodapro.get_cams( start_date=pd.Timestamp('2020-01-01'), end_date=pd.Timestamp('2020-05-04'), latitude=55.7906, From 128c3f9ae849c75dd026ccfed4bfd24621cb9eca Mon Sep 17 00:00:00 2001 From: AdamRJensen Date: Fri, 11 Jun 2021 12:31:08 -0500 Subject: [PATCH 56/57] Change start_date/end_date to start/end --- pvlib/iotools/sodapro.py | 24 ++++++++++++------------ pvlib/tests/iotools/test_sodapro.py | 20 ++++++++++---------- 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/pvlib/iotools/sodapro.py b/pvlib/iotools/sodapro.py index 8ea0e12066..05ce01dbda 100644 --- a/pvlib/iotools/sodapro.py +++ b/pvlib/iotools/sodapro.py @@ -41,10 +41,10 @@ '0 year 1 month 0 day 0 h 0 min 0 s': '1M'} -def get_cams(start_date, end_date, latitude, longitude, email, - identifier='mcclear', altitude=None, time_step='1h', - time_ref='UT', verbose=False, integrated=False, label=None, - map_variables=True, server='www.soda-is.com', timeout=30): +def get_cams(start, end, latitude, longitude, email, identifier='mcclear', + altitude=None, time_step='1h', time_ref='UT', verbose=False, + integrated=False, label=None, map_variables=True, + server='www.soda-is.com', timeout=30): """ Retrieve time-series of radiation and/or clear-sky global, beam, and diffuse radiation from CAMS. Data from CAMS Radiation [1]_ and CAMS McClear @@ -62,9 +62,9 @@ def get_cams(start_date, end_date, latitude, longitude, email, Parameters ---------- - start_date: datetime like + start: datetime like First day of the requested period - end_date: datetime like + end: datetime like Last day of the requested period latitude: float in decimal degrees, between -90 and 90, north is positive (ISO 19115) @@ -181,8 +181,8 @@ def get_cams(start_date, end_date, latitude, longitude, email, altitude = -999 # Start and end date should be in the format: yyyy-mm-dd - start_date = start_date.strftime('%Y-%m-%d') - end_date = end_date.strftime('%Y-%m-%d') + start = start.strftime('%Y-%m-%d') + end = end.strftime('%Y-%m-%d') email = email.replace('@', '%2540') # Format email address identifier = 'get_{}'.format(identifier.lower()) # Format identifier str @@ -193,8 +193,8 @@ def get_cams(start_date, end_date, latitude, longitude, email, 'latitude': latitude, 'longitude': longitude, 'altitude': altitude, - 'date_begin': start_date, - 'date_end': end_date, + 'date_begin': start, + 'date_end': end, 'time_ref': time_ref, 'summarization': time_step_str, 'username': email, @@ -286,13 +286,13 @@ def parse_cams(fbuf, integrated=False, label=None, map_variables=True): if k_new in ['latitude', 'longitude', 'altitude']: metadata[k_new] = float(metadata.pop(k_old)) - mmetadataeta['radiation_unit'] = \ + metadata['radiation_unit'] = \ {True: 'Wh/m^2', False: 'W/m^2'}[integrated] # Determine the time_step from the metadata dictionary time_step = SUMMATION_PERIOD_TO_TIME_STEP[ metadata['Summarization (integration) period']] - mmetadataeta['time_step'] = time_step + metadata['time_step'] = time_step data = pd.read_csv(fbuf, sep=';', comment='#', header=None, names=names) diff --git a/pvlib/tests/iotools/test_sodapro.py b/pvlib/tests/iotools/test_sodapro.py index 47cebdc01b..92a60e9403 100644 --- a/pvlib/tests/iotools/test_sodapro.py +++ b/pvlib/tests/iotools/test_sodapro.py @@ -216,8 +216,8 @@ def test_get_cams(requests_mock, testfile, index, columns, values, dtypes, # Make API call - an error is raised if requested URI does not match out, metadata = sodapro.get_cams( - start_date=pd.Timestamp('2020-01-01'), - end_date=pd.Timestamp('2020-05-04'), + start=pd.Timestamp('2020-01-01'), + end=pd.Timestamp('2020-05-04'), latitude=55.7906, longitude=12.5251, email='pvlib-admin@googlegroups.com', @@ -232,8 +232,8 @@ def test_get_cams(requests_mock, testfile, index, columns, values, dtypes, # Test if Warning is raised if verbose mode is True and time_step != '1min' with pytest.warns(UserWarning, match='Verbose mode only supports'): _ = sodapro.get_cams( - start_date=pd.Timestamp('2020-01-01'), - end_date=pd.Timestamp('2020-05-04'), + start=pd.Timestamp('2020-01-01'), + end=pd.Timestamp('2020-05-04'), latitude=55.7906, longitude=12.5251, email='pvlib-admin@googlegroups.com', @@ -263,8 +263,8 @@ def test_get_cams_bad_request(requests_mock): # In the below example a non-registrered email is specified with pytest.raises(requests.HTTPError, match='Failed to execute WPS'): _ = sodapro.get_cams( - start_date=pd.Timestamp('2020-01-01'), - end_date=pd.Timestamp('2020-05-04'), + start=pd.Timestamp('2020-01-01'), + end=pd.Timestamp('2020-05-04'), latitude=55.7906, longitude=12.5251, email='test@test.com', # a non-registrered email @@ -276,8 +276,8 @@ def test_get_cams_bad_request(requests_mock): # Test if value error is raised if incorrect identifier is specified with pytest.raises(ValueError, match='Identifier must be either'): _ = sodapro.get_cams( - start_date=pd.Timestamp('2020-01-01'), - end_date=pd.Timestamp('2020-05-04'), + start=pd.Timestamp('2020-01-01'), + end=pd.Timestamp('2020-05-04'), latitude=55.7906, longitude=12.5251, email='test@test.com', @@ -286,8 +286,8 @@ def test_get_cams_bad_request(requests_mock): # Test if value error is raised if incorrect time step is specified with pytest.raises(ValueError, match='Time step not recognized'): _ = sodapro.get_cams( - start_date=pd.Timestamp('2020-01-01'), - end_date=pd.Timestamp('2020-05-04'), + start=pd.Timestamp('2020-01-01'), + end=pd.Timestamp('2020-05-04'), latitude=55.7906, longitude=12.5251, email='test@test.com', From 1e6d262ac062c02343ab87b892135a23ac8e49c3 Mon Sep 17 00:00:00 2001 From: AdamRJensen Date: Fri, 11 Jun 2021 12:36:16 -0500 Subject: [PATCH 57/57] Make stickler happy --- pvlib/iotools/sodapro.py | 2 +- pvlib/tests/iotools/test_sodapro.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pvlib/iotools/sodapro.py b/pvlib/iotools/sodapro.py index 05ce01dbda..a27e6f1423 100644 --- a/pvlib/iotools/sodapro.py +++ b/pvlib/iotools/sodapro.py @@ -228,7 +228,7 @@ def get_cams(start, end, latitude, longitude, email, identifier='mcclear', elif res.headers['Content-Type'] == 'application/csv': fbuf = io.StringIO(res.content.decode('utf-8')) data, metadata = parse_cams(fbuf, integrated=integrated, label=label, - map_variables=map_variables) + map_variables=map_variables) return data, metadata diff --git a/pvlib/tests/iotools/test_sodapro.py b/pvlib/tests/iotools/test_sodapro.py index 92a60e9403..10f9a1e8c9 100644 --- a/pvlib/tests/iotools/test_sodapro.py +++ b/pvlib/tests/iotools/test_sodapro.py @@ -167,7 +167,7 @@ def generate_expected_dataframe(values, columns, index, dtypes): def test_read_cams(testfile, index, columns, values, dtypes): expected = generate_expected_dataframe(values, columns, index, dtypes) out, metadata = sodapro.read_cams(testfile, integrated=False, - map_variables=True) + map_variables=True) assert_frame_equal(out, expected, check_less_precise=True)