diff --git a/ci/azure/posix.yml b/ci/azure/posix.yml index cbcdf5694d..086f03dd69 100644 --- a/ci/azure/posix.yml +++ b/ci/azure/posix.yml @@ -23,7 +23,7 @@ jobs: versionSpec: '$(python.version)' - script: | - pip install pytest pytest-cov pytest-mock pytest-timeout pytest-azurepipelines pytest-rerunfailures pytest-remotedata + pip install pytest pytest-cov pytest-mock requests-mock pytest-timeout pytest-azurepipelines pytest-rerunfailures pytest-remotedata pip install -e . pytest pvlib --junitxml=junit/test-results.xml --cov=com --cov-report=xml --cov-report=html displayName: 'Test with pytest' diff --git a/ci/requirements-py36-min.yml b/ci/requirements-py36-min.yml index 29f63c1be1..84adcb360d 100644 --- a/ci/requirements-py36-min.yml +++ b/ci/requirements-py36-min.yml @@ -19,3 +19,4 @@ dependencies: - scipy==1.2.0 - pytest-rerunfailures # conda version is >3.6 - pytest-remotedata # conda package is 0.3.0, needs > 0.3.1 + - requests-mock diff --git a/ci/requirements-py36.yml b/ci/requirements-py36.yml index fb37fe8404..c49455119f 100644 --- a/ci/requirements-py36.yml +++ b/ci/requirements-py36.yml @@ -16,6 +16,7 @@ dependencies: - pytest - pytest-cov - pytest-mock + - requests-mock - pytest-rerunfailures - pytest-remotedata - pytest-timeout diff --git a/ci/requirements-py37.yml b/ci/requirements-py37.yml index 1542bb35d9..3203b004d1 100644 --- a/ci/requirements-py37.yml +++ b/ci/requirements-py37.yml @@ -16,6 +16,7 @@ dependencies: - pytest - pytest-cov - pytest-mock + - requests-mock - pytest-timeout - pytest-rerunfailures - pytest-remotedata diff --git a/ci/requirements-py38.yml b/ci/requirements-py38.yml index 6db508fd53..ca3a968335 100644 --- a/ci/requirements-py38.yml +++ b/ci/requirements-py38.yml @@ -16,6 +16,7 @@ dependencies: - pytest - pytest-cov - pytest-mock + - requests-mock - pytest-timeout - pytest-rerunfailures - pytest-remotedata diff --git a/ci/requirements-py39.yml b/ci/requirements-py39.yml index 35a3fa1952..16c6449158 100644 --- a/ci/requirements-py39.yml +++ b/ci/requirements-py39.yml @@ -16,6 +16,7 @@ dependencies: - pytest - pytest-cov - pytest-mock + - requests-mock - pytest-timeout - pytest-rerunfailures - pytest-remotedata diff --git a/docs/sphinx/source/api.rst b/docs/sphinx/source/api.rst index 735754f577..a9d57b926b 100644 --- a/docs/sphinx/source/api.rst +++ b/docs/sphinx/source/api.rst @@ -488,6 +488,9 @@ relevant to solar energy modeling. iotools.get_pvgis_tmy iotools.read_pvgis_tmy iotools.read_bsrn + iotools.get_cams + iotools.read_cams + iotools.parse_cams A :py:class:`~pvlib.location.Location` object may be created from metadata in some files. diff --git a/docs/sphinx/source/whatsnew/v0.9.0.rst b/docs/sphinx/source/whatsnew/v0.9.0.rst index aeb600b100..00b49ae250 100644 --- a/docs/sphinx/source/whatsnew/v0.9.0.rst +++ b/docs/sphinx/source/whatsnew/v0.9.0.rst @@ -105,6 +105,11 @@ Enhancements ~~~~~~~~~~~~ * Add :func:`~pvlib.iotools.read_bsrn` for reading BSRN solar radiation data files. (:pull:`1145`, :issue:`1015`) +* Add :func:`~pvlib.iotools.get_cams`, + :func:`~pvlib.iotools.parse_cams`, and + :func:`~pvlib.iotools.read_cams` + for retrieving, parsing, and reading CAMS Radiation and McClear time-series + files. (:pull:`1175`) * In :py:class:`~pvlib.modelchain.ModelChain`, attributes which contain output of models are now collected into ``ModelChain.results``. (:pull:`1076`, :issue:`1067`) diff --git a/pvlib/data/cams_mcclear_1min_verbose.csv b/pvlib/data/cams_mcclear_1min_verbose.csv new file mode 100644 index 0000000000..0bfb037b08 --- /dev/null +++ b/pvlib/data/cams_mcclear_1min_verbose.csv @@ -0,0 +1,60 @@ +# Coding: utf-8 +# File format version: 4 +# Title: CAMS McClear v3.1 model of clear-sky irradiation. +# Content: A time-series of solar radiation received on horizontal plane and plane always normal to the sun rays at ground level assuming clear sky. +# Calls on the McClear clear-sky model. Returns the global, beam and diffuse irradiations integrated over a selected time step, +# for a selected location (worldwide coverage) and a selected period. +# The research leading to these results has received funding from the European Union within the Copernicus programme. +# Provider: MINES ParisTech (France) +# More information at: http://www.soda-pro.com/web-services/radiation/cams-mcclear +# Date begin (ISO 8601): 2020-06-01T00:00:00.0 +# Date end (ISO 8601): 2020-06-03T00:00:00.0 +# Latitude (positive North, ISO 19115): 55.7906 +# Longitude (positive East, ISO 19115): 12.5251 +# Altitude (m): 39.00 +# Elevation of CAMS cell (m): 28.64 +# Time reference: Universal time (UT) +# +# Encoding partly from D2.8.III.13-14 INSPIRE Data Specification on Atmospheric Conditions and Meteorological Geographical Features - Technical Guidelines (2013-12-10) and CF (Climate and Forecast) metadata (2013-11-11) +# CF Standard Names registry of ObservablePropertyValue +# http://cfconventions.org/Data/cf-standard-names/27/build/cf-standard-name-table.html +# urn:x-inspire:specification:DS-AC-MF:observable-property-name:cf-standard-name:1.6 +# ObservableProperty +# basePhenomenon:"integral_of_surface_downwelling_shortwave_flux_in_air_assuming_clear_sky_wrt_time" +# uom:"Wh m-2" [unit] +# StatisticalMeasure +# statisticalFunction: "sum" +# Summarization (integration) period: 0 year 0 month 0 day 0 h 1 min 0 s +# noValue: nan +# File generated on: 2021-02-26 +# +# Columns: +# 1. Observation period (ISO 8601) +# 2. TOA. Irradiation on horizontal plane at the top of atmosphere (Wh/m2) +# 3. Clear sky GHI. Clear sky global irradiation on horizontal plane at ground level (Wh/m2) +# 4. Clear sky BHI. Clear sky beam irradiation on horizontal plane at ground level (Wh/m2) +# 5. Clear sky DHI. Clear sky diffuse irradiation on horizontal plane at ground level (Wh/m2) +# 6. Clear sky BNI. Clear sky beam irradiation on mobile plane following the sun at normal incidence (Wh/m2) +# 7. sza. Solar zenithal angle for the middle of the summarization (deg) +# 8. summer/winter split. 1.0 means summer, 0.0 means winter +# 9. tco3. Total column content of ozone (Dobson unit) +#10. tcwv. Total column content of water vapour (kg/m2) +#11. AOD BC. Partial aerosol optical depth at 550 nm for black carbon +#12. AOD DU. Partial aerosol optical depth at 550 nm for dust +#13. AOD SS. Partial aerosol optical depth at 550 nm for sea salt +#14. AOD OR. Partial aerosol optical depth at 550 nm for organic matter +#15. AOD SU. Partial aerosol optical depth at 550 nm for sulphate +#16. AOD NI. Partial aerosol optical depth at 550 nm for nitrate +#17. AOD AM. Partial aerosol optical depth at 550 nm for ammonium +#18. alpha. Angstroem coefficient for aerosol +#19. Aerosol type. Obsolete (value is always -1) +#20. fiso. MODIS-like BRDF parameter fiso +#21. fvol. MODIS-like BRDF parameter fvol +#22. fgeo. MODIS-like BRDF parameter fgeo +#23. albedo. Ground albedo +# +# Observation period;TOA;Clear sky GHI;Clear sky BHI;Clear sky DHI;Clear sky BNI;sza;summer/winter split;tco3;tcwv;AOD BC;AOD DU;AOD SS;AOD OR;AOD SU;AOD NI;AOD AM;alpha;Aerosol type;fiso;fvol;fgeo;albedo +2020-06-01T12:00:00.0/2020-06-01T12:01:00.0;18.0699;14.1417;12.5594;1.5823;15.3380;35.0308;0.9723;341.0221;17.7962;0.0065;0.0067;0.0008;0.0215;0.0252;0.0087;0.0022;nan;-1;0.1668;0.0912;0.0267;0.1359 +2020-06-01T12:01:00.0/2020-06-01T12:02:00.0;18.0584;14.1311;12.5484;1.5827;15.3343;35.0828;0.9723;341.0223;17.8020;0.0065;0.0067;0.0008;0.0215;0.0253;0.0087;0.0022;nan;-1;0.1668;0.0912;0.0267;0.1359 +2020-06-01T12:02:00.0/2020-06-01T12:03:00.0;18.0467;14.1204;12.5372;1.5831;15.3306;35.1357;0.9723;341.0224;17.8079;0.0065;0.0067;0.0008;0.0216;0.0253;0.0087;0.0022;nan;-1;0.1668;0.0912;0.0267;0.1359 +2020-06-01T12:03:00.0/2020-06-01T12:04:00.0;18.0348;14.1094;12.5259;1.5835;15.3269;35.1896;0.9723;341.0226;17.8137;0.0065;0.0067;0.0008;0.0217;0.0253;0.0087;0.0022;nan;-1;0.1668;0.0912;0.0267;0.1359 diff --git a/pvlib/data/cams_mcclear_monthly.csv b/pvlib/data/cams_mcclear_monthly.csv new file mode 100644 index 0000000000..70dfa99333 --- /dev/null +++ b/pvlib/data/cams_mcclear_monthly.csv @@ -0,0 +1,42 @@ +# Coding: utf-8 +# File format version: 2 +# Title: CAMS McClear v3.1 model of clear-sky irradiation. +# Content: A time-series of solar radiation received on horizontal plane and plane always normal to the sun rays at ground level assuming clear sky. +# Calls on the McClear clear-sky model. Returns the global, beam and diffuse irradiations integrated over a selected time step, +# for a selected location (worldwide coverage) and a selected period. +# The research leading to these results has received funding from the European Union within the Copernicus programme. +# Provider: MINES ParisTech (France) +# More information at: http://www.soda-pro.com/web-services/radiation/cams-mcclear +# Date begin (ISO 8601): 2020-01-01T00:00:00.0 +# Date end (ISO 8601): 2021-01-01T00:00:00.0 +# Latitude (positive North, ISO 19115): 55.7906 +# Longitude (positive East, ISO 19115): 12.5251 +# Altitude (m): 39.00 +# Time reference: Universal time (UT) +# +# Encoding partly from D2.8.III.13-14 INSPIRE Data Specification on Atmospheric Conditions and Meteorological Geographical Features - Technical Guidelines (2013-12-10) and CF (Climate and Forecast) metadata (2013-11-11) +# CF Standard Names registry of ObservablePropertyValue +# http://cfconventions.org/Data/cf-standard-names/27/build/cf-standard-name-table.html +# urn:x-inspire:specification:DS-AC-MF:observable-property-name:cf-standard-name:1.6 +# ObservableProperty +# basePhenomenon:"integral_of_surface_downwelling_shortwave_flux_in_air_assuming_clear_sky_wrt_time" +# uom:"Wh m-2" [unit] +# StatisticalMeasure +# statisticalFunction: "sum" +# Summarization (integration) period: 0 year 1 month 0 day 0 h 0 min 0 s +# noValue: nan +# File generated on: 2021-02-23 +# +# Columns: +# 1. Observation period (ISO 8601) +# 2. TOA. Irradiation on horizontal plane at the top of atmosphere (Wh/m2) +# 3. Clear sky GHI. Clear sky global irradiation on horizontal plane at ground level (Wh/m2) +# 4. Clear sky BHI. Clear sky beam irradiation on horizontal plane at ground level (Wh/m2) +# 5. Clear sky DHI. Clear sky diffuse irradiation on horizontal plane at ground level (Wh/m2) +# 6. Clear sky BNI. Clear sky beam irradiation on mobile plane following the sun at normal incidence (Wh/m2) +# +# Observation period;TOA;Clear sky GHI;Clear sky BHI;Clear sky DHI;Clear sky BNI +2020-01-01T00:00:00.0/2020-02-01T00:00:00.0;50168.9961;29424.7578;19492.6445;9932.1123;105764.1953 +2020-02-01T00:00:00.0/2020-03-01T00:00:00.0;91338.5234;59010.3047;40636.3008;18374.0020;140930.5781 +2020-03-01T00:00:00.0/2020-04-01T00:00:00.0;172855.2031;121402.9141;93124.6250;28278.2891;228798.9062 +2020-04-01T00:00:00.0/2020-05-01T00:00:00.0;248215.0312;180546.1406;142470.4688;38075.6680;279122.9375 diff --git a/pvlib/data/cams_radiation_1min_verbose.csv b/pvlib/data/cams_radiation_1min_verbose.csv new file mode 100644 index 0000000000..3686c86f4e --- /dev/null +++ b/pvlib/data/cams_radiation_1min_verbose.csv @@ -0,0 +1,72 @@ +# Coding: utf-8 +# File format version: 5 +# Title: CAMS Radiation Service v3.2 all-sky irradiation (derived from satellite data). +# Content: A time-series of solar radiation received on horizontal plane and plane always normal to the sun rays at ground level. +# Returns the global, beam and diffuse irradiations integrated over a selected time step, +# for a selected location (Meteosat Second Generation satellite coverage) and a selected period. +# The research leading to these results has received funding from the European Union within the Copernicus programme. +# Provider: MINES ParisTech (France) +# More information at: http://www.soda-pro.com/web-services/radiation/cams-radiation-service +# Date begin (ISO 8601): 2020-06-01T00:00:00.0 +# Date end (ISO 8601): 2020-06-02T00:00:00.0 +# Latitude (positive North, ISO 19115): 55.7906 +# Longitude (positive East, ISO 19115): 12.5251 +# Altitude (m): 39.00 +# Elevation of CAMS cell (m): 28.64 +# Time reference: Universal time (UT) +# +# Encoding partly from D2.8.III.13-14 INSPIRE Data Specification on Atmospheric Conditions and Meteorological Geographical Features - Technical Guidelines (2013-12-10) and CF (Climate and Forecast) metadata (2013-11-11) +# CF Standard Names registry of ObservablePropertyValue +# http://cfconventions.org/Data/cf-standard-names/27/build/cf-standard-name-table.html +# urn:x-inspire:specification:DS-AC-MF:observable-property-name:cf-standard-name:1.6 +# ObservableProperty +# basePhenomenon:"integral_of_surface_downwelling_shortwave_flux_in_air_sky_wrt_time" +# uom:"Wh m-2" [unit] +# StatisticalMeasure +# statisticalFunction: "sum" +# Summarization (integration) period: 0 year 0 month 0 day 0 h 1 min 0 s +# noValue: nan +# File generated on: 2021-03-07 +# +# Columns: +# 1. Observation period (ISO 8601) +# 2. TOA. Irradiation on horizontal plane at the top of atmosphere (Wh/m2) +# 3. Clear sky GHI. Clear sky global irradiation on horizontal plane at ground level (Wh/m2) +# 4. Clear sky BHI. Clear sky beam irradiation on horizontal plane at ground level (Wh/m2) +# 5. Clear sky DHI. Clear sky diffuse irradiation on horizontal plane at ground level (Wh/m2) +# 6. Clear sky BNI. Clear sky beam irradiation on mobile plane following the sun at normal incidence (Wh/m2) +# 7. GHI. Global irradiation on horizontal plane at ground level (Wh/m2) +# 8. BHI. Beam irradiation on horizontal plane at ground level (Wh/m2) +# 9. DHI. Diffuse irradiation on horizontal plane at ground level (Wh/m2) +#10. BNI. Beam irradiation on mobile plane following the sun at normal incidence (Wh/m2) +#11. Reliability. Proportion of reliable data in the summarization (0-1) +#12. sza. Solar zenithal angle for the middle of the summarization (deg) +#13. summer/winter split. 1.0 means summer, 0.0 means winter +#14. tco3. Total column content of ozone (Dobson unit) +#15. tcwv. Total column content of water vapour (kg/m2) +#16. AOD BC. Partial aerosol optical depth at 550 nm for black carbon +#17. AOD DU. Partial aerosol optical depth at 550 nm for dust +#18. AOD SS. Partial aerosol optical depth at 550 nm for sea salt +#19. AOD OR. Partial aerosol optical depth at 550 nm for organic matter +#20. AOD SU. Partial aerosol optical depth at 550 nm for sulphate +#21. AOD NI. Partial aerosol optical depth at 550 nm for nitrate +#22. AOD AM. Partial aerosol optical depth at 550 nm for ammonium +#23. alpha. Angstroem coefficient for aerosol +#24. Aerosol type. Obsolete (value is always -1) +#25. fiso. MODIS-like BRDF parameter fiso +#26. fvol. MODIS-like BRDF parameter fvol +#27. fgeo. MODIS-like BRDF parameter fgeo +#28. albedo. Ground albedo +#29. Cloud optical depth (value of the nearest acquisition time of the pixel) +#30. Cloud coverage of the pixel (percentage from 0 to 100, value of the nearest acquisition time of the pixel) -1=no value +#31. Cloud type (value of the nearest acquisition time of the pixel) -1=no value 0=no clouds 5=low-level cloud 6=medium-level cloud 7=high-level cloud 8=thin cloud +#32. GHI no corr. Global irradiation without bias correction on horizontal plane at ground level (Wh/m2) +#33. BHI no corr. Beam irradiation without bias correction on horizontal plane at ground level (Wh/m2) +#34. DHI no corr. Diffuse irradiation without bias correction on horizontal plane at ground level (Wh/m2) +#35. BNI no corr. Beam irradiation without bias correction on mobile plane following the sun at normal incidence (Wh/m2) +# +# Observation period;TOA;Clear sky GHI;Clear sky BHI;Clear sky DHI;Clear sky BNI;GHI;BHI;DHI;BNI;Reliability;sza;summer/winter split;tco3;tcwv;AOD BC;AOD DU;AOD SS;AOD OR;AOD SU;AOD NI;AOD AM;alpha;Aerosol type;fiso;fvol;fgeo;albedo;Cloud optical depth;Cloud coverage;Cloud type;GHI no corr;BHI no corr;DHI no corr;BNI no corr +2020-06-01T12:00:00.0/2020-06-01T12:01:00.0;18.0699;14.1417;12.5594;1.5823;15.3380;13.5893;11.7057;1.8837;14.2954;1.0000;35.0308;0.9723;341.0221;17.7962;0.0065;0.0067;0.0008;0.0215;0.0252;0.0087;0.0022;nan;-1;0.1668;0.0912;0.0267;0.1359;0.000000;0;5;14.1417;12.5594;1.5823;15.3380 +2020-06-01T12:01:00.0/2020-06-01T12:02:00.0;18.0584;14.1311;12.5484;1.5827;15.3343;13.5801;11.6955;1.8846;14.2920;1.0000;35.0828;0.9723;341.0223;17.8020;0.0065;0.0067;0.0008;0.0215;0.0253;0.0087;0.0022;nan;-1;0.1668;0.0912;0.0267;0.1359;0.000000;0;5;14.1311;12.5484;1.5827;15.3343 +2020-06-01T12:02:00.0/2020-06-01T12:03:00.0;18.0467;14.1204;12.5372;1.5831;15.3306;13.5697;11.6849;1.8848;14.2883;1.0000;35.1357;0.9723;341.0224;17.8079;0.0065;0.0067;0.0008;0.0216;0.0253;0.0087;0.0022;nan;-1;0.1668;0.0912;0.0267;0.1359;0.000000;0;5;14.1204;12.5372;1.5831;15.3306 +2020-06-01T12:03:00.0/2020-06-01T12:04:00.0;18.0348;14.1094;12.5259;1.5835;15.3269;13.5602;11.6744;1.8858;14.2849;1.0000;35.1896;0.9723;341.0226;17.8137;0.0065;0.0067;0.0008;0.0217;0.0253;0.0087;0.0022;nan;-1;0.1668;0.0912;0.0267;0.1359;0.000000;0;5;14.1094;12.5259;1.5835;15.3269 diff --git a/pvlib/data/cams_radiation_monthly.csv b/pvlib/data/cams_radiation_monthly.csv new file mode 100644 index 0000000000..0c9b9ba0b7 --- /dev/null +++ b/pvlib/data/cams_radiation_monthly.csv @@ -0,0 +1,47 @@ +# Coding: utf-8 +# File format version: 2 +# Title: CAMS Radiation Service v3.2 all-sky irradiation (derived from satellite data). +# Content: A time-series of solar radiation received on horizontal plane and plane always normal to the sun rays at ground level. +# Returns the global, beam and diffuse irradiations integrated over a selected time step, +# for a selected location (Meteosat Second Generation satellite coverage) and a selected period. +# The research leading to these results has received funding from the European Union within the Copernicus programme. +# Provider: MINES ParisTech (France) +# More information at: http://www.soda-pro.com/web-services/radiation/cams-radiation-service +# Date begin (ISO 8601): 2020-01-01T00:00:00.0 +# Date end (ISO 8601): 2021-02-01T00:00:00.0 +# Latitude (positive North, ISO 19115): 55.7906 +# Longitude (positive East, ISO 19115): 12.5251 +# Altitude (m): 39.00 +# Time reference: Universal time (UT) +# +# Encoding partly from D2.8.III.13-14 INSPIRE Data Specification on Atmospheric Conditions and Meteorological Geographical Features - Technical Guidelines (2013-12-10) and CF (Climate and Forecast) metadata (2013-11-11) +# CF Standard Names registry of ObservablePropertyValue +# http://cfconventions.org/Data/cf-standard-names/27/build/cf-standard-name-table.html +# urn:x-inspire:specification:DS-AC-MF:observable-property-name:cf-standard-name:1.6 +# ObservableProperty +# basePhenomenon:"integral_of_surface_downwelling_shortwave_flux_in_air_sky_wrt_time" +# uom:"Wh m-2" [unit] +# StatisticalMeasure +# statisticalFunction: "sum" +# Summarization (integration) period: 0 year 1 month 0 day 0 h 0 min 0 s +# noValue: nan +# File generated on: 2021-03-07 +# +# Columns: +# 1. Observation period (ISO 8601) +# 2. TOA. Irradiation on horizontal plane at the top of atmosphere (Wh/m2) +# 3. Clear sky GHI. Clear sky global irradiation on horizontal plane at ground level (Wh/m2) +# 4. Clear sky BHI. Clear sky beam irradiation on horizontal plane at ground level (Wh/m2) +# 5. Clear sky DHI. Clear sky diffuse irradiation on horizontal plane at ground level (Wh/m2) +# 6. Clear sky BNI. Clear sky beam irradiation on mobile plane following the sun at normal incidence (Wh/m2) +# 7. GHI. Global irradiation on horizontal plane at ground level (Wh/m2) +# 8. BHI. Beam irradiation on horizontal plane at ground level (Wh/m2) +# 9. DHI. Diffuse irradiation on horizontal plane at ground level (Wh/m2) +#10. BNI. Beam irradiation on mobile plane following the sun at normal incidence (Wh/m2) +#11. Reliability. Proportion of reliable data in the summarization (0-1) +# +# Observation period;TOA;Clear sky GHI;Clear sky BHI;Clear sky DHI;Clear sky BNI;GHI;BHI;DHI;BNI;Reliability +2020-01-01T00:00:00.0/2020-02-01T00:00:00.0;50169.1914;29424.8965;19492.7637;9932.1328;105764.5547;15531.9336;2568.7678;12972.1865;12469.0654;0.9970 +2020-02-01T00:00:00.0/2020-03-01T00:00:00.0;91338.7188;59010.4766;40636.4961;18373.9805;140931.0156;33073.7422;9732.8369;23351.6523;33306.4258;0.9956 +2020-03-01T00:00:00.0/2020-04-01T00:00:00.0;172855.4062;121403.0703;93124.7656;28278.3086;228799.0156;89403.4375;51798.5312;37620.5898;118724.5781;0.9949 +2020-04-01T00:00:00.0/2020-05-01T00:00:00.0;248215.1562;180546.2812;142470.6094;38075.6641;279123.0625;141625.0781;88746.7266;52930.9297;168456.6094;0.9897 diff --git a/pvlib/iotools/__init__.py b/pvlib/iotools/__init__.py index ba5d5e8807..b717c801ca 100644 --- a/pvlib/iotools/__init__.py +++ b/pvlib/iotools/__init__.py @@ -14,3 +14,6 @@ from pvlib.iotools.psm3 import parse_psm3 # noqa: F401 from pvlib.iotools.pvgis import get_pvgis_tmy, read_pvgis_tmy # noqa: F401 from pvlib.iotools.bsrn import read_bsrn # noqa: F401 +from pvlib.iotools.sodapro import get_cams # noqa: F401 +from pvlib.iotools.sodapro import read_cams # noqa: F401 +from pvlib.iotools.sodapro import parse_cams # noqa: F401 diff --git a/pvlib/iotools/sodapro.py b/pvlib/iotools/sodapro.py new file mode 100644 index 0000000000..a27e6f1423 --- /dev/null +++ b/pvlib/iotools/sodapro.py @@ -0,0 +1,377 @@ +"""Functions to access data from Copernicus Atmosphere Monitoring Service + (CAMS) radiation service. +.. codeauthor:: Adam R. Jensen +""" + +import pandas as pd +import requests +import io +import warnings + + +CAMS_INTEGRATED_COLUMNS = [ + 'TOA', 'Clear sky GHI', 'Clear sky BHI', 'Clear sky DHI', 'Clear sky BNI', + 'GHI', 'BHI', 'DHI', 'BNI', + 'GHI no corr', 'BHI no corr', 'DHI no corr', 'BNI no corr'] + +# Dictionary mapping CAMS Radiation and McClear variables to pvlib names +CAMS_VARIABLE_MAP = { + 'TOA': 'ghi_extra', + 'Clear sky GHI': 'ghi_clear', + 'Clear sky BHI': 'bhi_clear', + 'Clear sky DHI': 'dhi_clear', + 'Clear sky BNI': 'dni_clear', + 'GHI': 'ghi', + 'BHI': 'bhi', + 'DHI': 'dhi', + 'BNI': 'dni', + 'sza': 'solar_zenith', +} + +# Dictionary mapping time steps to CAMS time step format +TIME_STEPS_MAP = {'1min': 'PT01M', '15min': 'PT15M', '1h': 'PT01H', + '1d': 'P01D', '1M': 'P01M'} + +TIME_STEPS_IN_HOURS = {'1min': 1/60, '15min': 15/60, '1h': 1, '1d': 24} + +SUMMATION_PERIOD_TO_TIME_STEP = {'0 year 0 month 0 day 0 h 1 min 0 s': '1min', + '0 year 0 month 0 day 0 h 15 min 0 s': '15min', # noqa + '0 year 0 month 0 day 1 h 0 min 0 s': '1h', + '0 year 0 month 1 day 0 h 0 min 0 s': '1d', + '0 year 1 month 0 day 0 h 0 min 0 s': '1M'} + + +def get_cams(start, end, latitude, longitude, email, identifier='mcclear', + altitude=None, time_step='1h', time_ref='UT', verbose=False, + integrated=False, label=None, map_variables=True, + server='www.soda-is.com', timeout=30): + """ + Retrieve time-series of radiation and/or clear-sky global, beam, and + diffuse radiation from CAMS. Data from CAMS Radiation [1]_ and CAMS McClear + [2]_ are retrieved from SoDa [3]_. + + Time coverage: 2004-01-01 to two days ago + + Access: free, but requires registration, see [1]_ + + Requests: max. 100 per day + + Geographical coverage: Wordwide for CAMS McClear and -66° to 66° in both + latitude and longitude for CAMS Radiation + + + Parameters + ---------- + start: datetime like + First day of the requested period + end: datetime like + Last day of the requested period + latitude: float + in decimal degrees, between -90 and 90, north is positive (ISO 19115) + longitude : float + in decimal degrees, between -180 and 180, east is positive (ISO 19115) + email: str + Email address linked to a SoDa account + identifier: {'mcclear', 'cams_radiation'} + Specify whether to retrieve CAMS Radiation or McClear parameters + altitude: float, default: None + Altitude in meters. If None, then the altitude is determined from the + NASA SRTM database + time_step: str, {'1min', '15min', '1h', '1d', '1M'}, default: '1h' + Time step of the time series, either 1 minute, 15 minute, hourly, + daily, or monthly. + time_ref: str, {'UT', 'TST'}, default: 'UT' + 'UT' (universal time) or 'TST' (True Solar Time) + verbose: boolean, default: False + Verbose mode outputs additional parameters (aerosols). Only available + for 1 minute and universal time. See [1]_ for parameter description. + integrated: boolean, default False + Whether to return radiation parameters as integrated values (Wh/m^2) + or as average irradiance values (W/m^2) (pvlib preferred units) + label: {'right', 'left'}, default: None + Which bin edge label to label time-step with. The default is 'left' for + all time steps except for '1M' which has a default of 'right'. + map_variables: bool, default: True + When true, renames columns of the DataFrame to pvlib variable names + where applicable. See variable CAMS_VARIABLE_MAP. + server: str, default: 'www.soda-is.com' + Main server (www.soda-is.com) or backup mirror server (pro.soda-is.com) + timeout : int, default 30 + Time in seconds to wait for server response before timeout + + Returns + ------- + data: pandas.DataFrame + Timeseries data, see Notes for columns + metadata: dict + Metadata of the requested time-series + + Notes + ----- + In order to use the CAMS services, users must register for a free SoDa + account using an email address [1]_. + + The returned data DataFrame includes the following fields: + + ======================== ====== ========================================= + Key, mapped key Format Description + ======================== ====== ========================================= + **Mapped field names are returned when the map_variables argument is True** + --------------------------------------------------------------------------- + Observation period str Beginning/end of time period + TOA, ghi_extra float Horizontal radiation at top of atmosphere + Clear sky GHI, ghi_clear float Clear sky global radiation on horizontal + Clear sky BHI, bhi_clear float Clear sky beam radiation on horizontal + Clear sky DHI, dhi_clear float Clear sky diffuse radiation on horizontal + Clear sky BNI, dni_clear float Clear sky beam radiation normal to sun + GHI, ghi† float Global horizontal radiation + BHI, bhi† float Beam (direct) radiation on horizontal + DHI, dhi† float Diffuse horizontal radiation + BNI, dni† float Beam (direct) radiation normal to the sun + Reliability† float Reliable data fraction in summarization + ======================== ====== ========================================= + + †Parameters only returned if identifier='cams_radiation'. For description + of additional output parameters in verbose mode, see [1]_ and [2]_. + + Note that it is recommended to specify the latitude and longitude to at + least the fourth decimal place. + + Variables corresponding to standard pvlib variables are renamed, + e.g. `sza` becomes `solar_zenith`. See the + `pvlib.iotools.cams.CAMS_VARIABLE_MAP` dict for the complete + mapping. + + See Also + -------- + pvlib.iotools.read_cams, pvlib.iotools.parse_cams + + Raises + ------ + requests.HTTPError + If the request is invalid, then an XML file is returned by the CAMS + service and the error message will be raised as an exception. + + References + ---------- + .. [1] `CAMS Radiation Service Info + `_ + .. [2] `CAMS McClear Service Info + `_ + .. [3] `CAMS McClear Automatic Access + `_ + """ + try: + time_step_str = TIME_STEPS_MAP[time_step] + except KeyError: + raise ValueError(f'Time step not recognized. Must be one of ' + f'{list(TIME_STEPS_MAP.keys())}') + + if (verbose) and ((time_step != '1min') or (time_ref != 'UT')): + verbose = False + warnings.warn("Verbose mode only supports 1 min. UT time series!") + + if identifier not in ['mcclear', 'cams_radiation']: + raise ValueError('Identifier must be either mcclear or cams_radiation') + + # Format verbose variable to the required format: {'true', 'false'} + verbose = str(verbose).lower() + + if altitude is None: # Let SoDa get elevation from the NASA SRTM database + altitude = -999 + + # Start and end date should be in the format: yyyy-mm-dd + start = start.strftime('%Y-%m-%d') + end = end.strftime('%Y-%m-%d') + + email = email.replace('@', '%2540') # Format email address + identifier = 'get_{}'.format(identifier.lower()) # Format identifier str + + base_url = f"http://{server}/service/wps" + + data_inputs_dict = { + 'latitude': latitude, + 'longitude': longitude, + 'altitude': altitude, + 'date_begin': start, + 'date_end': end, + 'time_ref': time_ref, + 'summarization': time_step_str, + 'username': email, + 'verbose': verbose} + + # Manual formatting of the input parameters seperating each by a semicolon + data_inputs = ";".join([f"{key}={value}" for key, value in + data_inputs_dict.items()]) + + params = {'Service': 'WPS', + 'Request': 'Execute', + 'Identifier': identifier, + 'version': '1.0.0', + 'RawDataOutput': 'irradiation', + } + + # The DataInputs parameter of the URL has to be manually formatted and + # added to the base URL as it contains sub-parameters seperated by + # semi-colons, which gets incorrectly formatted by the requests function + # if passed using the params argument. + res = requests.get(base_url + '?DataInputs=' + data_inputs, params=params, + timeout=timeout) + + # Invalid requests returns an XML error message and the HTTP staus code 200 + # as if the request was successful. Therefore, errors cannot be handled + # automatic (e.g. res.raise_for_status()) and errors are handled manually + if res.headers['Content-Type'] == 'application/xml': + errors = res.text.split('ows:ExceptionText')[1][1:-2] + raise requests.HTTPError(errors, response=res) + # Successful requests returns a csv data file + elif res.headers['Content-Type'] == 'application/csv': + fbuf = io.StringIO(res.content.decode('utf-8')) + data, metadata = parse_cams(fbuf, integrated=integrated, label=label, + map_variables=map_variables) + return data, metadata + + +def parse_cams(fbuf, integrated=False, label=None, map_variables=True): + """ + Parse a file-like buffer with data in the format of a CAMS Radiation or + McClear file. The CAMS services are described in [1]_ and [2]_. + + Parameters + ---------- + fbuf: file-like object + File-like object containing data to read. + integrated: boolean, default False + Whether to return radiation parameters as integrated values (Wh/m^2) + or as average irradiance values (W/m^2) (pvlib preferred units) + label: {'right', 'left'}, default: None + Which bin edge label to label time-step with. The default is 'left' for + all time steps except for '1M' which has a default of 'right'. + map_variables: bool, default: True + When true, renames columns of the Dataframe to pvlib variable names + where applicable. See variable CAMS_VARIABLE_MAP. + + Returns + ------- + data: pandas.DataFrame + Timeseries data from CAMS Radiation or McClear + metadata: dict + Metadata available in the file. + + See Also + -------- + pvlib.iotools.read_cams, pvlib.iotools.get_cams + + References + ---------- + .. [1] `CAMS Radiation Service Info + `_ + .. [2] `CAMS McClear Service Info + `_ + """ + metadata = {} + # Initial lines starting with # contain metadata + while True: + line = fbuf.readline().rstrip('\n') + if line.startswith('# Observation period'): + # The last line of the metadata section contains the column names + names = line.lstrip('# ').split(';') + break # End of metadata section has been reached + elif ': ' in line: + metadata[line.split(': ')[0].lstrip('# ')] = line.split(': ')[1] + + # Convert latitude, longitude, and altitude values from strings to floats + for k_old in list(metadata.keys()): + k_new = k_old.lstrip().split(' ')[0].lower() + if k_new in ['latitude', 'longitude', 'altitude']: + metadata[k_new] = float(metadata.pop(k_old)) + + metadata['radiation_unit'] = \ + {True: 'Wh/m^2', False: 'W/m^2'}[integrated] + + # Determine the time_step from the metadata dictionary + time_step = SUMMATION_PERIOD_TO_TIME_STEP[ + metadata['Summarization (integration) period']] + metadata['time_step'] = time_step + + data = pd.read_csv(fbuf, sep=';', comment='#', header=None, names=names) + + obs_period = data['Observation period'].str.split('/') + + # Set index as the start observation time (left) and localize to UTC + if (label == 'left') | ((label is None) & (time_step != '1M')): + data.index = pd.to_datetime(obs_period.str[0], utc=True) + # Set index as the stop observation time (right) and localize to UTC + # default label for monthly data is 'right' following Pandas' convention + elif (label == 'right') | ((label is None) & (time_step == '1M')): + data.index = pd.to_datetime(obs_period.str[1], utc=True) + + # For time_steps '1d' and '1M', drop timezone and round to nearest midnight + if (time_step == '1d') | (time_step == '1M'): + data.index = pd.DatetimeIndex(data.index.date) + # For monthly data with 'right' label, the index should be the last + # date of the month and not the first date of the following month + if (time_step == '1M') & (label != 'left'): + data.index = data.index - pd.Timedelta(days=1) + + if not integrated: # Convert radiation values from Wh/m2 to W/m2 + integrated_cols = [c for c in CAMS_INTEGRATED_COLUMNS + if c in data.columns] + + if time_step == '1M': + time_delta = (pd.to_datetime(obs_period.str[1]) + - pd.to_datetime(obs_period.str[0])) + hours = time_delta.dt.total_seconds()/60/60 + data[integrated_cols] = data[integrated_cols].\ + divide(hours.tolist(), axis='rows') + else: + data[integrated_cols] = (data[integrated_cols] / + TIME_STEPS_IN_HOURS[time_step]) + data.index.name = None # Set index name to None + if map_variables: + data = data.rename(columns=CAMS_VARIABLE_MAP) + + return data, metadata + + +def read_cams(filename, integrated=False, label=None, map_variables=True): + """ + Read a CAMS Radiation or McClear file into a pandas DataFrame. CAMS + radiation and McClear are described in [1]_ and [2]_, respectively. + + Parameters + ---------- + filename: str + Filename of a file containing data to read. + integrated: boolean, default False + Whether to return radiation parameters as integrated values (Wh/m^2) + or as average irradiance values (W/m^2) (pvlib preferred units) + label: {'right', 'left}, default: None + Which bin edge label to label time-step with. The default is 'left' for + all time steps except for '1M' which has a default of 'right'. + map_variables: bool, default: True + When true, renames columns of the Dataframe to pvlib variable names + where applicable. See variable CAMS_VARIABLE_MAP. + + Returns + ------- + data: pandas.DataFrame + Timeseries data from CAMS Radiation or McClear + :func:`pvlib.iotools.get_cams` for fields + metadata: dict + Metadata available in the file. + + See Also + -------- + pvlib.iotools.parse_cams, pvlib.iotools.get_cams + + References + ---------- + .. [1] `CAMS Radiation Service Info + `_ + .. [2] `CAMS McClear Service Info + `_ + """ + with open(str(filename), 'r') as fbuf: + content = parse_cams(fbuf, integrated, label, map_variables) + return content diff --git a/pvlib/tests/iotools/test_sodapro.py b/pvlib/tests/iotools/test_sodapro.py new file mode 100644 index 0000000000..10f9a1e8c9 --- /dev/null +++ b/pvlib/tests/iotools/test_sodapro.py @@ -0,0 +1,296 @@ +""" +test iotools for sodapro +""" + +import pandas as pd +import numpy as np +import requests +import pytest + +from pvlib.iotools import sodapro +from ..conftest import DATA_DIR, assert_frame_equal + + +testfile_mcclear_verbose = DATA_DIR / 'cams_mcclear_1min_verbose.csv' +testfile_mcclear_monthly = DATA_DIR / 'cams_mcclear_monthly.csv' +testfile_radiation_verbose = DATA_DIR / 'cams_radiation_1min_verbose.csv' +testfile_radiation_monthly = DATA_DIR / 'cams_radiation_monthly.csv' + + +index_verbose = pd.date_range('2020-06-01 12', periods=4, freq='1T', tz='UTC') +index_monthly = pd.date_range('2020-01-01', periods=4, freq='1M') + + +dtypes_mcclear_verbose = [ + 'object', 'float64', 'float64', 'float64', 'float64', 'float64', 'float64', + 'float64', 'float64', 'float64', 'float64', 'float64', 'float64', + 'float64', 'float64', 'float64', 'float64', 'float64', 'int64', 'float64', + 'float64', 'float64', 'float64'] + +dtypes_mcclear = [ + 'object', 'float64', 'float64', 'float64', 'float64', 'float64'] + +dtypes_radiation_verbose = [ + 'object', 'float64', 'float64', 'float64', 'float64', 'float64', 'float64', + 'float64', 'float64', 'float64', 'float64', 'float64', 'float64', + 'float64', 'float64', 'float64', 'float64', 'float64', 'float64', + 'float64', 'float64', 'float64', 'float64', 'int64', 'float64', 'float64', + 'float64', 'float64', 'float64', 'int64', 'int64', 'float64', 'float64', + 'float64', 'float64'] + +dtypes_radiation = [ + 'object', 'float64', 'float64', 'float64', 'float64', 'float64', 'float64', + 'float64', 'float64', 'float64', 'float64'] + + +columns_mcclear_verbose = [ + 'Observation period', 'ghi_extra', 'ghi_clear', 'bhi_clear', + 'dhi_clear', 'dni_clear', 'solar_zenith', 'summer/winter split', 'tco3', + 'tcwv', 'AOD BC', 'AOD DU', 'AOD SS', 'AOD OR', 'AOD SU', 'AOD NI', + 'AOD AM', 'alpha', 'Aerosol type', 'fiso', 'fvol', 'fgeo', 'albedo'] + +columns_mcclear = [ + 'Observation period', 'ghi_extra', 'ghi_clear', 'bhi_clear', 'dhi_clear', + 'dni_clear'] + +columns_radiation_verbose = [ + 'Observation period', 'ghi_extra', 'ghi_clear', 'bhi_clear', 'dhi_clear', + 'dni_clear', 'ghi', 'bhi', 'dhi', 'dni', 'Reliability', 'solar_zenith', + 'summer/winter split', 'tco3', 'tcwv', 'AOD BC', 'AOD DU', 'AOD SS', + 'AOD OR', 'AOD SU', 'AOD NI', 'AOD AM', 'alpha', 'Aerosol type', 'fiso', + 'fvol', 'fgeo', 'albedo', 'Cloud optical depth', 'Cloud coverage', + 'Cloud type', 'GHI no corr', 'BHI no corr', 'DHI no corr', 'BNI no corr'] + +columns_radiation_verbose_unmapped = [ + 'Observation period', 'TOA', 'Clear sky GHI', 'Clear sky BHI', + 'Clear sky DHI', 'Clear sky BNI', 'GHI', 'BHI', 'DHI', 'BNI', + 'Reliability', 'sza', 'summer/winter split', 'tco3', 'tcwv', 'AOD BC', + 'AOD DU', 'AOD SS', 'AOD OR', 'AOD SU', 'AOD NI', 'AOD AM', 'alpha', + 'Aerosol type', 'fiso', 'fvol', 'fgeo', 'albedo', 'Cloud optical depth', + 'Cloud coverage', 'Cloud type', 'GHI no corr', 'BHI no corr', + 'DHI no corr', 'BNI no corr'] + +columns_radiation = [ + 'Observation period', 'ghi_extra', 'ghi_clear', 'bhi_clear', 'dhi_clear', + 'dni_clear', 'ghi', 'bhi', 'dhi', 'dni', 'Reliability'] + + +values_mcclear_verbose = np.array([ + ['2020-06-01T12:00:00.0/2020-06-01T12:01:00.0', 1084.194, 848.5020, + 753.564, 94.938, 920.28, 35.0308, 0.9723, 341.0221, 17.7962, 0.0065, + 0.0067, 0.0008, 0.0215, 0.0252, 0.0087, 0.0022, np.nan, -1, 0.1668, + 0.0912, 0.0267, 0.1359], + ['2020-06-01T12:01:00.0/2020-06-01T12:02:00.0', 1083.504, 847.866, 752.904, + 94.962, 920.058, 35.0828, 0.9723, 341.0223, 17.802, 0.0065, 0.0067, + 0.0008, 0.0215, 0.0253, 0.0087, 0.0022, np.nan, -1, 0.1668, 0.0912, + 0.0267, 0.1359], + ['2020-06-01T12:02:00.0/2020-06-01T12:03:00.0', 1082.802, 847.224, 752.232, + 94.986, 919.836, 35.1357, 0.9723, 341.0224, 17.8079, 0.0065, 0.0067, + 0.0008, 0.0216, 0.0253, 0.0087, 0.0022, np.nan, -1, 0.1668, 0.0912, + 0.0267, 0.1359], + ['2020-06-01T12:03:00.0/2020-06-01T12:04:00.0', 1082.088, 846.564, 751.554, + 95.01, 919.614, 35.1896, 0.9723, 341.0226, 17.8137, 0.0065, 0.0067, + 0.0008, 0.0217, 0.0253, 0.0087, 0.0022, np.nan, -1, 0.1668, 0.0912, + 0.0267, 0.1359]]) + +values_mcclear_monthly = np.array([ + ['2020-01-01T00:00:00.0/2020-02-01T00:00:00.0', 67.4314, 39.5494, + 26.1998, 13.3496, 142.1562], + ['2020-02-01T00:00:00.0/2020-03-01T00:00:00.0', 131.2335, 84.7849, + 58.3855, 26.3994, 202.4865], + ['2020-03-01T00:00:00.0/2020-04-01T00:00:00.0', 232.3323, 163.176, + 125.1675, 38.0085, 307.5254], + ['2020-04-01T00:00:00.0/2020-05-01T00:00:00.0', 344.7431, 250.7585, + 197.8757, 52.8829, 387.6707]]) + +values_radiation_verbose = np.array([ + ['2020-06-01T12:00:00.0/2020-06-01T12:01:00.0', 1084.194, 848.502, 753.564, + 94.938, 920.28, 815.358, 702.342, 113.022, 857.724, 1.0, 35.0308, 0.9723, + 341.0221, 17.7962, 0.0065, 0.0067, 0.0008, 0.0215, 0.0252, 0.0087, 0.0022, + np.nan, -1, 0.1668, 0.0912, 0.0267, 0.1359, 0.0, 0, 5, 848.502, 753.564, + 94.938, 920.28], + ['2020-06-01T12:01:00.0/2020-06-01T12:02:00.0', 1083.504, 847.866, 752.904, + 94.962, 920.058, 814.806, 701.73, 113.076, 857.52, 1.0, 35.0828, 0.9723, + 341.0223, 17.802, 0.0065, 0.0067, 0.0008, 0.0215, 0.0253, 0.0087, 0.0022, + np.nan, -1, 0.1668, 0.0912, 0.0267, 0.1359, 0.0, 0, 5, 847.866, 752.904, + 94.962, 920.058], + ['2020-06-01T12:02:00.0/2020-06-01T12:03:00.0', 1082.802, 847.224, 752.232, + 94.986, 919.836, 814.182, 701.094, 113.088, 857.298, 1.0, 35.1357, 0.9723, + 341.0224, 17.8079, 0.0065, 0.0067, 0.0008, 0.0216, 0.0253, 0.0087, 0.0022, + np.nan, -1, 0.1668, 0.0912, 0.0267, 0.1359, 0.0, 0, 5, 847.224, 752.232, + 94.986, 919.836], + ['2020-06-01T12:03:00.0/2020-06-01T12:04:00.0', 1082.088, 846.564, 751.554, + 95.01, 919.614, 813.612, 700.464, 113.148, 857.094, 1.0, 35.1896, 0.9723, + 341.0226, 17.8137, 0.0065, 0.0067, 0.0008, 0.0217, 0.0253, 0.0087, 0.0022, + np.nan, -1, 0.1668, 0.0912, 0.0267, 0.1359, 0.0, 0, 5, 846.564, 751.554, + 95.01, 919.614]]) + +values_radiation_verbose_integrated = np.copy(values_radiation_verbose) +values_radiation_verbose_integrated[:, 1:10] = \ + values_radiation_verbose_integrated[:, 1:10].astype(float)/60 +values_radiation_verbose_integrated[:, 31:35] = \ + values_radiation_verbose_integrated[:, 31:35].astype(float)/60 + +values_radiation_monthly = np.array([ + ['2020-01-01T00:00:00.0/2020-02-01T00:00:00.0', 67.4317, 39.5496, + 26.2, 13.3496, 142.1567, 20.8763, 3.4526, 17.4357, 16.7595, 0.997], + ['2020-02-01T00:00:00.0/2020-03-01T00:00:00.0', 131.2338, 84.7852, + 58.3858, 26.3994, 202.4871, 47.5197, 13.984, 33.5512, 47.8541, 0.9956], + ['2020-03-01T00:00:00.0/2020-04-01T00:00:00.0', 232.3325, 163.1762, + 125.1677, 38.0085, 307.5256, 120.1659, 69.6217, 50.5653, 159.576, 0.9949], + ['2020-04-01T00:00:00.0/2020-05-01T00:00:00.0', 344.7433, 250.7587, + 197.8758, 52.8829, 387.6709, 196.7015, 123.2593, 73.5152, 233.9675, + 0.9897]]) + + +# @pytest.fixture +def generate_expected_dataframe(values, columns, index, dtypes): + """Create dataframe from arrays of values, columns and index, in order to + use this dataframe to compare to. + """ + expected = pd.DataFrame(values, columns=columns, index=index) + expected.index.freq = None + for (col, _dtype) in zip(expected.columns, dtypes): + expected[col] = expected[col].astype(_dtype) + return expected + + +@pytest.mark.parametrize('testfile,index,columns,values,dtypes', [ + (testfile_mcclear_verbose, index_verbose, columns_mcclear_verbose, + values_mcclear_verbose, dtypes_mcclear_verbose), + (testfile_mcclear_monthly, index_monthly, columns_mcclear, + values_mcclear_monthly, dtypes_mcclear), + (testfile_radiation_verbose, index_verbose, columns_radiation_verbose, + values_radiation_verbose, dtypes_radiation_verbose), + (testfile_radiation_monthly, index_monthly, columns_radiation, + values_radiation_monthly, dtypes_radiation)]) +def test_read_cams(testfile, index, columns, values, dtypes): + expected = generate_expected_dataframe(values, columns, index, dtypes) + out, metadata = sodapro.read_cams(testfile, integrated=False, + map_variables=True) + assert_frame_equal(out, expected, check_less_precise=True) + + +def test_read_cams_integrated_unmapped_label(): + # Default label is 'left' for 1 minute time resolution, hence 1 minute is + # added for label='right' + expected = generate_expected_dataframe( + values_radiation_verbose_integrated, + columns_radiation_verbose_unmapped, + index_verbose+pd.Timedelta(minutes=1), dtypes=dtypes_radiation_verbose) + out, metadata = sodapro.read_cams(testfile_radiation_verbose, + integrated=True, label='right', + map_variables=False) + assert_frame_equal(out, expected, check_less_precise=True) + + +def test_read_cams_metadata(): + _, metadata = sodapro.read_cams(testfile_mcclear_monthly, integrated=False) + assert metadata['Time reference'] == 'Universal time (UT)' + assert metadata['noValue'] == 'nan' + assert metadata['latitude'] == 55.7906 + assert metadata['longitude'] == 12.5251 + assert metadata['altitude'] == 39.0 + assert metadata['radiation_unit'] == 'W/m^2' + assert metadata['time_step'] == '1M' + + +@pytest.mark.parametrize('testfile,index,columns,values,dtypes,identifier', [ + (testfile_mcclear_monthly, index_monthly, columns_mcclear, + values_mcclear_monthly, dtypes_mcclear, 'mcclear'), + (testfile_radiation_monthly, index_monthly, columns_radiation, + values_radiation_monthly, dtypes_radiation, 'cams_radiation')]) +def test_get_cams(requests_mock, testfile, index, columns, values, dtypes, + identifier): + """Test that get_cams generates the correct URI request and that parse_cams + is being called correctly""" + # Open local test file containing McClear mothly data + with open(testfile, 'r') as test_file: + mock_response = test_file.read() + # Specify the full URI of a specific example, this ensures that all of the + # inputs are passing on correctly + url_test_cams = f'http://www.soda-is.com/service/wps?DataInputs=latitude=55.7906;longitude=12.5251;altitude=80;date_begin=2020-01-01;date_end=2020-05-04;time_ref=UT;summarization=P01M;username=pvlib-admin%2540googlegroups.com;verbose=false&Service=WPS&Request=Execute&Identifier=get_{identifier}&version=1.0.0&RawDataOutput=irradiation' # noqa: E501 + + requests_mock.get(url_test_cams, text=mock_response, + headers={'Content-Type': 'application/csv'}) + + # Make API call - an error is raised if requested URI does not match + out, metadata = sodapro.get_cams( + start=pd.Timestamp('2020-01-01'), + end=pd.Timestamp('2020-05-04'), + latitude=55.7906, + longitude=12.5251, + email='pvlib-admin@googlegroups.com', + identifier=identifier, + altitude=80, + time_step='1M', + verbose=False, + integrated=False) + expected = generate_expected_dataframe(values, columns, index, dtypes) + assert_frame_equal(out, expected, check_less_precise=True) + + # Test if Warning is raised if verbose mode is True and time_step != '1min' + with pytest.warns(UserWarning, match='Verbose mode only supports'): + _ = sodapro.get_cams( + start=pd.Timestamp('2020-01-01'), + end=pd.Timestamp('2020-05-04'), + latitude=55.7906, + longitude=12.5251, + email='pvlib-admin@googlegroups.com', + identifier=identifier, + altitude=80, + time_step='1M', + verbose=True) + + +def test_get_cams_bad_request(requests_mock): + """Test that a the correct errors/warnings ares raised for invalid + requests inputs. Also tests if the specified server url gets used""" + + # Subset of an xml file returned for errornous requests + mock_response_bad = """ + + Failed to execute WPS process [get_mcclear]: + Please, register yourself at www.soda-pro.com + """ + + url_cams_bad_request = 'http://pro.soda-is.com/service/wps?DataInputs=latitude=55.7906;longitude=12.5251;altitude=-999;date_begin=2020-01-01;date_end=2020-05-04;time_ref=TST;summarization=PT01H;username=test%2540test.com;verbose=false&Service=WPS&Request=Execute&Identifier=get_mcclear&version=1.0.0&RawDataOutput=irradiation' # noqa: E501 + + requests_mock.get(url_cams_bad_request, text=mock_response_bad, + headers={'Content-Type': 'application/xml'}) + + # Test if HTTPError is raised if incorrect input is specified + # In the below example a non-registrered email is specified + with pytest.raises(requests.HTTPError, match='Failed to execute WPS'): + _ = sodapro.get_cams( + start=pd.Timestamp('2020-01-01'), + end=pd.Timestamp('2020-05-04'), + latitude=55.7906, + longitude=12.5251, + email='test@test.com', # a non-registrered email + identifier='mcclear', + time_ref='TST', + verbose=False, + time_step='1h', + server='pro.soda-is.com') + # Test if value error is raised if incorrect identifier is specified + with pytest.raises(ValueError, match='Identifier must be either'): + _ = sodapro.get_cams( + start=pd.Timestamp('2020-01-01'), + end=pd.Timestamp('2020-05-04'), + latitude=55.7906, + longitude=12.5251, + email='test@test.com', + identifier='test', # incorrect identifier + server='pro.soda-is.com') + # Test if value error is raised if incorrect time step is specified + with pytest.raises(ValueError, match='Time step not recognized'): + _ = sodapro.get_cams( + start=pd.Timestamp('2020-01-01'), + end=pd.Timestamp('2020-05-04'), + latitude=55.7906, + longitude=12.5251, + email='test@test.com', + identifier='mcclear', + time_step='test', # incorrect time step + server='pro.soda-is.com') diff --git a/setup.py b/setup.py index a876f0e995..216dc34a28 100755 --- a/setup.py +++ b/setup.py @@ -49,7 +49,8 @@ INSTALL_REQUIRES.append('dataclasses') TESTS_REQUIRE = ['nose', 'pytest', 'pytest-cov', 'pytest-mock', - 'pytest-timeout', 'pytest-rerunfailures', 'pytest-remotedata'] + 'requests-mock', 'pytest-timeout', 'pytest-rerunfailures', + 'pytest-remotedata'] EXTRAS_REQUIRE = { 'optional': ['cython', 'ephem', 'netcdf4', 'nrel-pysam', 'numba', 'pvfactors', 'siphon', 'statsmodels', 'tables',