From 100d1f42aa41ceb2e24d4e93fdf9b0202ece9092 Mon Sep 17 00:00:00 2001 From: Silvana Ovaitt Date: Thu, 22 Sep 2022 11:59:48 -0600 Subject: [PATCH 01/20] sam iotools #1371 adds a sam.py, and pytest --- pvlib/iotools/__init__.py | 1 + pvlib/iotools/sam.py | 164 ++++++++++++++++++++++++++++++++ pvlib/tests/iotools/test_sam.py | 29 ++++++ 3 files changed, 194 insertions(+) create mode 100644 pvlib/iotools/sam.py create mode 100644 pvlib/tests/iotools/test_sam.py diff --git a/pvlib/iotools/__init__.py b/pvlib/iotools/__init__.py index b02ce243ae..0b5231ac57 100644 --- a/pvlib/iotools/__init__.py +++ b/pvlib/iotools/__init__.py @@ -21,3 +21,4 @@ from pvlib.iotools.sodapro import get_cams # noqa: F401 from pvlib.iotools.sodapro import read_cams # noqa: F401 from pvlib.iotools.sodapro import parse_cams # noqa: F401 +from pvlib.iotools.sam import saveSAM_WeatherFile, tz_convert # noqa: F401 diff --git a/pvlib/iotools/sam.py b/pvlib/iotools/sam.py new file mode 100644 index 0000000000..abb4041c44 --- /dev/null +++ b/pvlib/iotools/sam.py @@ -0,0 +1,164 @@ +"""Functions for reading and writing SAM data files.""" + +import pandas as pd + +def saveSAM_WeatherFile(data, metadata, savefile='SAM_WeatherFile.csv', standardSAM = True, includeminute=False): + """ + Saves a dataframe with weather data from pvlib format on SAM-friendly format. + + Parameters + ----------- + data : pandas.DataFrame + timeseries data in PVLib format. Should be TZ converted (not UTC). Ideally it is one sequential year data; if not suggested to use standardSAM = False. + metdata : dictionary + Dictionary with at least 'latitude', 'longitude', 'elevation', 'source', and 'TZ' for timezone. + savefile : str + Name of file to save output as. + standardSAM : boolean + This checks the dataframe to avoid having a leap day, then averages it to SAM style (closed to the right), + and fills the years so it starst on YEAR/1/1 0:0 and ends on YEAR/12/31 23:00. + includeminute ; Bool + For hourly data, if SAM input does not have Minutes, it calculates the sun position 30 minutes + prior to the hour (i.e. 12 timestamp means sun position at 11:30) + If minutes are included, it will calculate the sun position at the time of the timestamp (12:00 at 12:00) + Set to true if resolution of data is sub-hourly. + + Returns + ------- + Nothing, it just writes the file. + + """ + + def _is_leap_and_29Feb(s): + ''' Creates a mask to help remove Leap Years. Obtained from: + https://stackoverflow.com/questions/34966422/remove-leap-year-day-from-pandas-dataframe/34966636 + ''' + return (s.index.year % 4 == 0) & \ + ((s.index.year % 100 != 0) | (s.index.year % 400 == 0)) & \ + (s.index.month == 2) & (s.index.day == 29) + + def _averageSAMStyle(df, interval='60T', closed='right', label='right'): + ''' Averages subhourly data into hourly data in SAM's expected format. + ''' + try: + df = df.resample(interval, closed=closed, label=label).mean() # + except: + print('Warning - unable to average') + return df + + def _fillYearSAMStyle(df, freq='60T'): + ''' Fills year + ''' + # add zeros for the rest of the year + if freq is None: + try: + freq = pd.infer_freq(df.index) + except: + freq = '60T' # 15 minute data by default + # add a timepoint at the end of the year + # idx = df.index + # apply correct TZ info (if applicable) + tzinfo = df.index.tzinfo + starttime = pd.to_datetime('%s-%s-%s %s:%s' % (df.index.year[0],1,1,0,0 ) ).tz_localize(tzinfo) + endtime = pd.to_datetime('%s-%s-%s %s:%s' % (df.index.year[-1],12,31,23,60-int(freq[:-1])) ).tz_localize(tzinfo) + + df2 = _averageSAMStyle(df, freq) + df2.iloc[0] = 0 # set first datapt to zero to forward fill w zeros + df2.iloc[-1] = 0 # set last datapt to zero to forward fill w zeros + df2.loc[starttime] = 0 + df2.loc[endtime] = 0 + df2 = df2.resample(freq).ffill() + return df2 + + + # Modify this to cut into different years. Right now handles partial year and sub-hourly interval. + if standardSAM: + filterdatesLeapYear = ~(_is_leap_and_29Feb(data)) + data = data[filterdatesLeapYear] + data = _fillYearSAMStyle(data) + + + # metadata + latitude=metadata['latitude'] + longitude=metadata['longitude'] + elevation=metadata['elevation'] + timezone_offset = metadata['TZ'] + source = metadata['source'] + + # make a header + header = '\n'.join( + [ 'Source,Latitude,Longitude,Time Zone,Elevation', + source + ',' + str(latitude) + ',' + str(longitude) + ',' + str(timezone_offset) + ',' + str(elevation)]) + '\n' + + savedata = pd.DataFrame({'Year':data.index.year, 'Month':data.index.month, 'Day':data.index.day, + 'Hour':data.index.hour}) + + if includeminute: + savedata['Minute'] = data.index.minute + + windspeed = list(data.wind_speed) + temp_amb = list(data.temp_air) + savedata['Wspd'] = windspeed + savedata['Tdry'] = temp_amb + + if 'dni' in data: + dni = list(data.dni) + savedata['DHI'] = dni + + if 'dhi' in data: + dhi = list(data.dhi) + savedata['DNI'] = dhi + + if 'ghi' in data: + ghi = list(data.ghi) + savedata['GHI'] = ghi + + if 'poa' in data: # This is a nifty function of SAM for using field measured POA irradiance! + poa = list(data.poa) + savedata['POA'] = poa + + if 'albedo' in data: + albedo = list(data.albedo) + savedata['Albedo'] = albedo + + with open(savefile, 'w', newline='') as ict: + # Write the header lines, including the index variable for + # the last one if you're letting Pandas produce that for you. + # (see above). + for line in header: + ict.write(line) + + savedata.to_csv(ict, index=False) + + +def tz_convert(df, tz_convert_val, metadata=None): + """ + Support function to convert metdata to a different local timezone. Particularly for + GIS weather files which are returned in UTC by default. + + Parameters + ---------- + df : DataFrame + A dataframe in UTC timezone + tz_convert_val : int + Convert timezone to this fixed value, following ISO standard + (negative values indicating West of UTC.) + Returns: metdata, metadata + + + Returns + ------- + df : DataFrame + Dataframe in the converted local timezone. + metadata : dict + Adds (or updates) the existing Timezone in the metadata dictionary + + """ + import pytz + if (type(tz_convert_val) == int) | (type(tz_convert_val) == float): + df = df.tz_convert(pytz.FixedOffset(tz_convert_val*60)) + + if metadata is not None: + metadata['TZ'] = tz_convert_val + return df, metadata + return df \ No newline at end of file diff --git a/pvlib/tests/iotools/test_sam.py b/pvlib/tests/iotools/test_sam.py new file mode 100644 index 0000000000..95346fe554 --- /dev/null +++ b/pvlib/tests/iotools/test_sam.py @@ -0,0 +1,29 @@ +""" +test the SAM IO tools +""" +import pandas as pd +from pvlib.iotools import get_pvgis_tmy, read_pvgis_hourly +from pvlib.iotools import saveSAM_WeatherFile, tz_convert +from ..conftest import (DATA_DIR, RERUNS, RERUNS_DELAY, assert_frame_equal, + fail_on_pvlib_version) + +# PVGIS Hourly tests +# The test files are actual files from PVGIS where the data section have been +# reduced to only a few lines +testfile_radiation_csv = DATA_DIR / \ + 'pvgis_hourly_Timeseries_45.000_8.000_SA_30deg_0deg_2016_2016.csv' + +# REMOVE +testfile_radiation_csv = r'C:\Users\sayala\Documents\GitHub\pvlib-python\pvlib\data\pvgis_hourly_Timeseries_45.000_8.000_SA_30deg_0deg_2016_2016.csv' + +def test_saveSAM_WeatherFile(): + data, inputs, metadata = read_pvgis_hourly(testfile_radiation_csv, map_variables=True)#, pvgis_format=pvgis_format) + metadata = {'latitude': inputs['latitude'], + 'longitude': inputs['longitude'], + 'elevation': inputs['elevation'], + 'source': 'User-generated'} + metadata['TZ'] = -7 + data = tz_convert(data, tz_convert_val=metadata['TZ']) + coerce_year=2021 # read_pvgis_hourly does not coerce_year, so doing it here. + data.index = data.index.map(lambda dt: dt.replace(year=coerce_year)) + saveSAM_WeatherFile(data, metadata, savefile='test_SAMWeatherFile.csv', standardSAM=True) From dfd5aa9777b45cd3d8ce68a2ba0aa4282df9a201 Mon Sep 17 00:00:00 2001 From: Silvana Ovaitt Date: Thu, 22 Sep 2022 12:04:16 -0600 Subject: [PATCH 02/20] add to reference --- docs/sphinx/source/reference/iotools.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/sphinx/source/reference/iotools.rst b/docs/sphinx/source/reference/iotools.rst index 514aeac2f5..5af92429f5 100644 --- a/docs/sphinx/source/reference/iotools.rst +++ b/docs/sphinx/source/reference/iotools.rst @@ -37,6 +37,8 @@ of sources and file formats relevant to solar energy modeling. iotools.get_cams iotools.read_cams iotools.parse_cams + iotools.saveSAM_WeatherFile + iotools.tz_convert A :py:class:`~pvlib.location.Location` object may be created from metadata in some files. From 1de291bf43b47e2fcfea29dc6739a23d1a325926 Mon Sep 17 00:00:00 2001 From: Silvana Ovaitt Date: Thu, 22 Sep 2022 12:08:05 -0600 Subject: [PATCH 03/20] update the latest whatsnew that I found --- docs/sphinx/source/whatsnew/v0.9.4.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/sphinx/source/whatsnew/v0.9.4.rst b/docs/sphinx/source/whatsnew/v0.9.4.rst index ea89248743..da1d29d58d 100644 --- a/docs/sphinx/source/whatsnew/v0.9.4.rst +++ b/docs/sphinx/source/whatsnew/v0.9.4.rst @@ -9,7 +9,8 @@ Deprecations Enhancements ~~~~~~~~~~~~ - +* :py:func:`pvlib.iotools.saveSAM_WeatherFile` now writes pvlib formated dataframe +into a weatherfile usable in SAM. (:pull:`1556`, :issue:`1371`) Bug fixes ~~~~~~~~~ @@ -33,3 +34,4 @@ Requirements Contributors ~~~~~~~~~~~~ +* Silvana Ovaitt (:ghuser:`shirubana`) \ No newline at end of file From 2de5a3716272da5b4e8c7ae20dbb762767f2b093 Mon Sep 17 00:00:00 2001 From: Silvana Ovaitt Date: Thu, 22 Sep 2022 12:33:23 -0600 Subject: [PATCH 04/20] stickler --- pvlib/iotools/sam.py | 102 ++++++++++++++++++-------------- pvlib/tests/iotools/test_sam.py | 14 +++-- 2 files changed, 66 insertions(+), 50 deletions(-) diff --git a/pvlib/iotools/sam.py b/pvlib/iotools/sam.py index abb4041c44..d2e325e2c9 100644 --- a/pvlib/iotools/sam.py +++ b/pvlib/iotools/sam.py @@ -2,52 +2,61 @@ import pandas as pd -def saveSAM_WeatherFile(data, metadata, savefile='SAM_WeatherFile.csv', standardSAM = True, includeminute=False): +def saveSAM_WeatherFile(data, metadata, savefile='SAM_WeatherFile.csv', + standardSAM=True, includeminute=False): """ - Saves a dataframe with weather data from pvlib format on SAM-friendly format. + Saves dataframe with weather data from pvlib format on SAM-friendly format. Parameters ----------- data : pandas.DataFrame - timeseries data in PVLib format. Should be TZ converted (not UTC). Ideally it is one sequential year data; if not suggested to use standardSAM = False. + timeseries data in PVLib format. Should be TZ converted (not UTC). + Ideally it is one sequential year data; if not suggested to use + standardSAM = False. metdata : dictionary - Dictionary with at least 'latitude', 'longitude', 'elevation', 'source', and 'TZ' for timezone. + Dictionary with 'latitude', 'longitude', 'elevation', 'source', + and 'TZ' for timezone. savefile : str - Name of file to save output as. + Name of file to save output as. standardSAM : boolean - This checks the dataframe to avoid having a leap day, then averages it to SAM style (closed to the right), - and fills the years so it starst on YEAR/1/1 0:0 and ends on YEAR/12/31 23:00. + This checks the dataframe to avoid having a leap day, then averages it + to SAM style (closed to the right), + and fills the years so it starst on YEAR/1/1 0:0 and ends on + YEAR/12/31 23:00. includeminute ; Bool - For hourly data, if SAM input does not have Minutes, it calculates the sun position 30 minutes - prior to the hour (i.e. 12 timestamp means sun position at 11:30) - If minutes are included, it will calculate the sun position at the time of the timestamp (12:00 at 12:00) + For hourly data, if SAM input does not have Minutes, it calculates the + sun position 30 minutes prior to the hour (i.e. 12 timestamp means sun + position at 11:30). + If minutes are included, it will calculate the sun position at the time + of the timestamp (12:00 at 12:00) Set to true if resolution of data is sub-hourly. Returns ------- Nothing, it just writes the file. - + """ def _is_leap_and_29Feb(s): - ''' Creates a mask to help remove Leap Years. Obtained from: - https://stackoverflow.com/questions/34966422/remove-leap-year-day-from-pandas-dataframe/34966636 + ''' Creates a mask to help remove Leap Years. Obtained from: + https://stackoverflow.com/questions/34966422/remove-leap-year-day- + from-pandas-dataframe/34966636 ''' return (s.index.year % 4 == 0) & \ ((s.index.year % 100 != 0) | (s.index.year % 400 == 0)) & \ (s.index.month == 2) & (s.index.day == 29) def _averageSAMStyle(df, interval='60T', closed='right', label='right'): - ''' Averages subhourly data into hourly data in SAM's expected format. + ''' Averages subhourly data into hourly data in SAM's expected format. ''' try: - df = df.resample(interval, closed=closed, label=label).mean() # + df = df.resample(interval, closed=closed, label=label).mean() except: print('Warning - unable to average') return df def _fillYearSAMStyle(df, freq='60T'): - ''' Fills year + ''' Fills year ''' # add zeros for the rest of the year if freq is None: @@ -59,38 +68,41 @@ def _fillYearSAMStyle(df, freq='60T'): # idx = df.index # apply correct TZ info (if applicable) tzinfo = df.index.tzinfo - starttime = pd.to_datetime('%s-%s-%s %s:%s' % (df.index.year[0],1,1,0,0 ) ).tz_localize(tzinfo) - endtime = pd.to_datetime('%s-%s-%s %s:%s' % (df.index.year[-1],12,31,23,60-int(freq[:-1])) ).tz_localize(tzinfo) + starttime = pd.to_datetime('%s-%s-%s %s:%s' % (df.index.year[0], 1, 1, + 0, 0)).tz_localize(tzinfo) + endtime = pd.to_datetime('%s-%s-%s %s:%s' % (df.index.year[-1], 12, 31, + 23, 60-int(freq[:-1])) + ).tz_localize(tzinfo) df2 = _averageSAMStyle(df, freq) df2.iloc[0] = 0 # set first datapt to zero to forward fill w zeros df2.iloc[-1] = 0 # set last datapt to zero to forward fill w zeros df2.loc[starttime] = 0 df2.loc[endtime] = 0 - df2 = df2.resample(freq).ffill() + df2 = df2.resample(freq).ffill() return df2 - - # Modify this to cut into different years. Right now handles partial year and sub-hourly interval. + # Modify this to cut into different years. Right now handles partial year + # and sub-hourly interval. if standardSAM: - filterdatesLeapYear = ~(_is_leap_and_29Feb(data)) + filterdatesLeapYear = ~(_is_leap_and_29Feb(data)) data = data[filterdatesLeapYear] data = _fillYearSAMStyle(data) - # metadata - latitude=metadata['latitude'] - longitude=metadata['longitude'] - elevation=metadata['elevation'] + latitude = metadata['latitude'] + longitude = metadata['longitude'] + elevation = metadata['elevation'] timezone_offset = metadata['TZ'] source = metadata['source'] - - # make a header - header = '\n'.join( - [ 'Source,Latitude,Longitude,Time Zone,Elevation', - source + ',' + str(latitude) + ',' + str(longitude) + ',' + str(timezone_offset) + ',' + str(elevation)]) + '\n' - savedata = pd.DataFrame({'Year':data.index.year, 'Month':data.index.month, 'Day':data.index.day, + # make a header + header = '\n'.join([ 'Source,Latitude,Longitude,Time Zone,Elevation', + source + ',' + str(latitude) + ',' + str(longitude) + + ',' + str(timezone_offset) + ',' + str(elevation)])+'\n' + + savedata = pd.DataFrame({'Year':data.index.year, 'Month':data.index.month, + 'Day':data.index.day, 'Hour':data.index.hour}) if includeminute: @@ -100,24 +112,24 @@ def _fillYearSAMStyle(df, freq='60T'): temp_amb = list(data.temp_air) savedata['Wspd'] = windspeed savedata['Tdry'] = temp_amb - + if 'dni' in data: dni = list(data.dni) savedata['DHI'] = dni - + if 'dhi' in data: dhi = list(data.dhi) savedata['DNI'] = dhi - + if 'ghi' in data: ghi = list(data.ghi) savedata['GHI'] = ghi - - if 'poa' in data: # This is a nifty function of SAM for using field measured POA irradiance! + + if 'poa' in data: poa = list(data.poa) savedata['POA'] = poa - - if 'albedo' in data: + + if 'albedo' in data: albedo = list(data.albedo) savedata['Albedo'] = albedo @@ -133,18 +145,18 @@ def _fillYearSAMStyle(df, freq='60T'): def tz_convert(df, tz_convert_val, metadata=None): """ - Support function to convert metdata to a different local timezone. Particularly for - GIS weather files which are returned in UTC by default. - + Support function to convert metdata to a different local timezone. + Particularly for GIS weather files which are returned in UTC by default. + Parameters ---------- df : DataFrame A dataframe in UTC timezone tz_convert_val : int - Convert timezone to this fixed value, following ISO standard + Convert timezone to this fixed value, following ISO standard (negative values indicating West of UTC.) - Returns: metdata, metadata - + Returns: metdata, metadata + Returns ------- diff --git a/pvlib/tests/iotools/test_sam.py b/pvlib/tests/iotools/test_sam.py index 95346fe554..7d7cdd378c 100644 --- a/pvlib/tests/iotools/test_sam.py +++ b/pvlib/tests/iotools/test_sam.py @@ -17,12 +17,16 @@ testfile_radiation_csv = r'C:\Users\sayala\Documents\GitHub\pvlib-python\pvlib\data\pvgis_hourly_Timeseries_45.000_8.000_SA_30deg_0deg_2016_2016.csv' def test_saveSAM_WeatherFile(): - data, inputs, metadata = read_pvgis_hourly(testfile_radiation_csv, map_variables=True)#, pvgis_format=pvgis_format) - metadata = {'latitude': inputs['latitude'], - 'longitude': inputs['longitude'], - 'elevation': inputs['elevation'], + data, months_selected, inputs, metadata = get_pvgis_tmy(latitude=33, longitude=-110, map_variables=True) + # Reading local file doesn't work read_pvgis_hourly returns different + # keys than get_pvgis_tmy when map=variables = True. Opened issue for that +# data, inputs, metadata = read_pvgis_hourly(testfile_radiation_csv, +# map_variables=True) + metadata = {'latitude': inputs['location']['latitude'], + 'longitude': inputs['location']['longitude'], + 'elevation': inputs['location']['elevation'], 'source': 'User-generated'} - metadata['TZ'] = -7 + metadata['TZ'] = -7 data = tz_convert(data, tz_convert_val=metadata['TZ']) coerce_year=2021 # read_pvgis_hourly does not coerce_year, so doing it here. data.index = data.index.map(lambda dt: dt.replace(year=coerce_year)) From 19b05bd1c1931ff1e8f2a1c1c31512d9aebc2269 Mon Sep 17 00:00:00 2001 From: Silvana Ovaitt Date: Thu, 22 Sep 2022 13:05:54 -0600 Subject: [PATCH 05/20] stickler take 2 --- pvlib/iotools/sam.py | 34 ++++++++++++++++----------------- pvlib/tests/iotools/test_sam.py | 22 ++++++++++----------- 2 files changed, 27 insertions(+), 29 deletions(-) diff --git a/pvlib/iotools/sam.py b/pvlib/iotools/sam.py index d2e325e2c9..9001d26a11 100644 --- a/pvlib/iotools/sam.py +++ b/pvlib/iotools/sam.py @@ -2,6 +2,7 @@ import pandas as pd + def saveSAM_WeatherFile(data, metadata, savefile='SAM_WeatherFile.csv', standardSAM=True, includeminute=False): """ @@ -49,10 +50,7 @@ def _is_leap_and_29Feb(s): def _averageSAMStyle(df, interval='60T', closed='right', label='right'): ''' Averages subhourly data into hourly data in SAM's expected format. ''' - try: - df = df.resample(interval, closed=closed, label=label).mean() - except: - print('Warning - unable to average') + df = df.resample(interval, closed=closed, label=label).mean() return df def _fillYearSAMStyle(df, freq='60T'): @@ -60,19 +58,17 @@ def _fillYearSAMStyle(df, freq='60T'): ''' # add zeros for the rest of the year if freq is None: - try: - freq = pd.infer_freq(df.index) - except: - freq = '60T' # 15 minute data by default + freq = pd.infer_freq(df.index) # add a timepoint at the end of the year # idx = df.index # apply correct TZ info (if applicable) tzinfo = df.index.tzinfo starttime = pd.to_datetime('%s-%s-%s %s:%s' % (df.index.year[0], 1, 1, - 0, 0)).tz_localize(tzinfo) + 0, 0) + ).tz_localize(tzinfo) endtime = pd.to_datetime('%s-%s-%s %s:%s' % (df.index.year[-1], 12, 31, 23, 60-int(freq[:-1])) - ).tz_localize(tzinfo) + ).tz_localize(tzinfo) df2 = _averageSAMStyle(df, freq) df2.iloc[0] = 0 # set first datapt to zero to forward fill w zeros @@ -97,13 +93,15 @@ def _fillYearSAMStyle(df, freq='60T'): source = metadata['source'] # make a header - header = '\n'.join([ 'Source,Latitude,Longitude,Time Zone,Elevation', - source + ',' + str(latitude) + ',' + str(longitude) - + ',' + str(timezone_offset) + ',' + str(elevation)])+'\n' + header = '\n'.join(['Source,Latitude,Longitude,Time Zone,Elevation', + source + ',' + str(latitude) + ',' + str(longitude) + + ',' + str(timezone_offset) + ',' + + str(elevation)])+'\n' - savedata = pd.DataFrame({'Year':data.index.year, 'Month':data.index.month, - 'Day':data.index.day, - 'Hour':data.index.hour}) + savedata = pd.DataFrame({'Year': data.index.year, + 'Month': data.index.month, + 'Day': data.index.day, + 'Hour': data.index.hour}) if includeminute: savedata['Minute'] = data.index.minute @@ -145,7 +143,7 @@ def _fillYearSAMStyle(df, freq='60T'): def tz_convert(df, tz_convert_val, metadata=None): """ - Support function to convert metdata to a different local timezone. + Support function to convert metdata to a different local timezone. Particularly for GIS weather files which are returned in UTC by default. Parameters @@ -173,4 +171,4 @@ def tz_convert(df, tz_convert_val, metadata=None): if metadata is not None: metadata['TZ'] = tz_convert_val return df, metadata - return df \ No newline at end of file + return df diff --git a/pvlib/tests/iotools/test_sam.py b/pvlib/tests/iotools/test_sam.py index 7d7cdd378c..7cc00a6a66 100644 --- a/pvlib/tests/iotools/test_sam.py +++ b/pvlib/tests/iotools/test_sam.py @@ -13,21 +13,21 @@ testfile_radiation_csv = DATA_DIR / \ 'pvgis_hourly_Timeseries_45.000_8.000_SA_30deg_0deg_2016_2016.csv' -# REMOVE -testfile_radiation_csv = r'C:\Users\sayala\Documents\GitHub\pvlib-python\pvlib\data\pvgis_hourly_Timeseries_45.000_8.000_SA_30deg_0deg_2016_2016.csv' - def test_saveSAM_WeatherFile(): - data, months_selected, inputs, metadata = get_pvgis_tmy(latitude=33, longitude=-110, map_variables=True) - # Reading local file doesn't work read_pvgis_hourly returns different + data, months_selected, inputs, metadata = get_pvgis_tmy(latitude=33, + longitude=-110, + map_variables=True) + # Reading local file doesn't work read_pvgis_hourly returns different # keys than get_pvgis_tmy when map=variables = True. Opened issue for that -# data, inputs, metadata = read_pvgis_hourly(testfile_radiation_csv, +# data, inputs, metadata = read_pvgis_hourly(testfile_radiation_csv, # map_variables=True) metadata = {'latitude': inputs['location']['latitude'], - 'longitude': inputs['location']['longitude'], - 'elevation': inputs['location']['elevation'], - 'source': 'User-generated'} + 'longitude': inputs['location']['longitude'], + 'elevation': inputs['location']['elevation'], + 'source': 'User-generated'} metadata['TZ'] = -7 data = tz_convert(data, tz_convert_val=metadata['TZ']) - coerce_year=2021 # read_pvgis_hourly does not coerce_year, so doing it here. + coerce_year = 2021 data.index = data.index.map(lambda dt: dt.replace(year=coerce_year)) - saveSAM_WeatherFile(data, metadata, savefile='test_SAMWeatherFile.csv', standardSAM=True) + saveSAM_WeatherFile(data, metadata, savefile='test_SAMWeatherFile.csv', + standardSAM=True) From c573344c97576a9b2ebfa218a6c8830396099df5 Mon Sep 17 00:00:00 2001 From: Silvana Ovaitt Date: Thu, 22 Sep 2022 13:11:37 -0600 Subject: [PATCH 06/20] stickler take 3 --- pvlib/iotools/sam.py | 2 +- pvlib/tests/iotools/test_sam.py | 17 ++++++----------- 2 files changed, 7 insertions(+), 12 deletions(-) diff --git a/pvlib/iotools/sam.py b/pvlib/iotools/sam.py index 9001d26a11..2909eb6e9d 100644 --- a/pvlib/iotools/sam.py +++ b/pvlib/iotools/sam.py @@ -3,7 +3,7 @@ import pandas as pd -def saveSAM_WeatherFile(data, metadata, savefile='SAM_WeatherFile.csv', +def saveSAM_WeatherFile(data, metadata, savefile='SAM_WeatherFile.csv', standardSAM=True, includeminute=False): """ Saves dataframe with weather data from pvlib format on SAM-friendly format. diff --git a/pvlib/tests/iotools/test_sam.py b/pvlib/tests/iotools/test_sam.py index 7cc00a6a66..679da90511 100644 --- a/pvlib/tests/iotools/test_sam.py +++ b/pvlib/tests/iotools/test_sam.py @@ -1,11 +1,9 @@ """ test the SAM IO tools """ -import pandas as pd -from pvlib.iotools import get_pvgis_tmy, read_pvgis_hourly +from pvlib.iotools import get_pvgis_tmy from pvlib.iotools import saveSAM_WeatherFile, tz_convert -from ..conftest import (DATA_DIR, RERUNS, RERUNS_DELAY, assert_frame_equal, - fail_on_pvlib_version) +from ..conftest import DATA_DIR # PVGIS Hourly tests # The test files are actual files from PVGIS where the data section have been @@ -13,14 +11,11 @@ testfile_radiation_csv = DATA_DIR / \ 'pvgis_hourly_Timeseries_45.000_8.000_SA_30deg_0deg_2016_2016.csv' + def test_saveSAM_WeatherFile(): - data, months_selected, inputs, metadata = get_pvgis_tmy(latitude=33, - longitude=-110, + data, months_selected, inputs, metadata = get_pvgis_tmy(latitude=33, + longitude=-110, map_variables=True) - # Reading local file doesn't work read_pvgis_hourly returns different - # keys than get_pvgis_tmy when map=variables = True. Opened issue for that -# data, inputs, metadata = read_pvgis_hourly(testfile_radiation_csv, -# map_variables=True) metadata = {'latitude': inputs['location']['latitude'], 'longitude': inputs['location']['longitude'], 'elevation': inputs['location']['elevation'], @@ -29,5 +24,5 @@ def test_saveSAM_WeatherFile(): data = tz_convert(data, tz_convert_val=metadata['TZ']) coerce_year = 2021 data.index = data.index.map(lambda dt: dt.replace(year=coerce_year)) - saveSAM_WeatherFile(data, metadata, savefile='test_SAMWeatherFile.csv', + saveSAM_WeatherFile(data, metadata, savefile='test_SAMWeatherFile.csv', standardSAM=True) From d9672e977485c81705298763548ae8f5ac417600 Mon Sep 17 00:00:00 2001 From: Silvana Ovaitt Date: Thu, 22 Sep 2022 14:06:27 -0600 Subject: [PATCH 07/20] codecov fix --- pvlib/iotools/sam.py | 5 +++++ pvlib/tests/iotools/test_sam.py | 8 +++++--- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/pvlib/iotools/sam.py b/pvlib/iotools/sam.py index 2909eb6e9d..9e61206795 100644 --- a/pvlib/iotools/sam.py +++ b/pvlib/iotools/sam.py @@ -130,6 +130,11 @@ def _fillYearSAMStyle(df, freq='60T'): if 'albedo' in data: albedo = list(data.albedo) savedata['Albedo'] = albedo + + # Not elegant but seems to work for the standardSAM format + if standardSAM and savedata.Albedo.iloc[0] == 0: + savedata.loc[savedata.index[0],'Albedo'] = savedata.loc[savedata.index[1]]['Albedo'] + savedata.loc[savedata.index[-1],'Albedo'] = savedata.loc[savedata.index[-2]]['Albedo'] with open(savefile, 'w', newline='') as ict: # Write the header lines, including the index variable for diff --git a/pvlib/tests/iotools/test_sam.py b/pvlib/tests/iotools/test_sam.py index 679da90511..f54afd6ced 100644 --- a/pvlib/tests/iotools/test_sam.py +++ b/pvlib/tests/iotools/test_sam.py @@ -4,6 +4,7 @@ from pvlib.iotools import get_pvgis_tmy from pvlib.iotools import saveSAM_WeatherFile, tz_convert from ..conftest import DATA_DIR +import numpy as np # PVGIS Hourly tests # The test files are actual files from PVGIS where the data section have been @@ -20,9 +21,10 @@ def test_saveSAM_WeatherFile(): 'longitude': inputs['location']['longitude'], 'elevation': inputs['location']['elevation'], 'source': 'User-generated'} - metadata['TZ'] = -7 - data = tz_convert(data, tz_convert_val=metadata['TZ']) + data, metadata = tz_convert(data, tz_convert_val=-7, metadata=metadata) + data['albedo'] = 0.2 coerce_year = 2021 + data['poa'] = np.nan data.index = data.index.map(lambda dt: dt.replace(year=coerce_year)) saveSAM_WeatherFile(data, metadata, savefile='test_SAMWeatherFile.csv', - standardSAM=True) + standardSAM=True, includeminute = True) From 98407a6b3196a7dac842bb6943cc87003fcd3525 Mon Sep 17 00:00:00 2001 From: Silvana Ovaitt Date: Fri, 23 Sep 2022 07:56:24 -0600 Subject: [PATCH 08/20] back with the stickler --- pvlib/iotools/sam.py | 11 +++++------ pvlib/tests/iotools/test_sam.py | 4 ++-- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/pvlib/iotools/sam.py b/pvlib/iotools/sam.py index 9e61206795..638a5d4911 100644 --- a/pvlib/iotools/sam.py +++ b/pvlib/iotools/sam.py @@ -57,10 +57,7 @@ def _fillYearSAMStyle(df, freq='60T'): ''' Fills year ''' # add zeros for the rest of the year - if freq is None: - freq = pd.infer_freq(df.index) # add a timepoint at the end of the year - # idx = df.index # apply correct TZ info (if applicable) tzinfo = df.index.tzinfo starttime = pd.to_datetime('%s-%s-%s %s:%s' % (df.index.year[0], 1, 1, @@ -130,11 +127,13 @@ def _fillYearSAMStyle(df, freq='60T'): if 'albedo' in data: albedo = list(data.albedo) savedata['Albedo'] = albedo - + # Not elegant but seems to work for the standardSAM format if standardSAM and savedata.Albedo.iloc[0] == 0: - savedata.loc[savedata.index[0],'Albedo'] = savedata.loc[savedata.index[1]]['Albedo'] - savedata.loc[savedata.index[-1],'Albedo'] = savedata.loc[savedata.index[-2]]['Albedo'] + savedata.loc[savedata.index[0], 'Albedo'] = savedata.loc[ + savedata.index[1]]['Albedo'] + savedata.loc[savedata.index[-1], 'Albedo'] = savedata.loc[ + savedata.index[-2]]['Albedo'] with open(savefile, 'w', newline='') as ict: # Write the header lines, including the index variable for diff --git a/pvlib/tests/iotools/test_sam.py b/pvlib/tests/iotools/test_sam.py index f54afd6ced..e92dc7bd30 100644 --- a/pvlib/tests/iotools/test_sam.py +++ b/pvlib/tests/iotools/test_sam.py @@ -22,9 +22,9 @@ def test_saveSAM_WeatherFile(): 'elevation': inputs['location']['elevation'], 'source': 'User-generated'} data, metadata = tz_convert(data, tz_convert_val=-7, metadata=metadata) - data['albedo'] = 0.2 + data['albedo'] = 0.2 coerce_year = 2021 data['poa'] = np.nan data.index = data.index.map(lambda dt: dt.replace(year=coerce_year)) saveSAM_WeatherFile(data, metadata, savefile='test_SAMWeatherFile.csv', - standardSAM=True, includeminute = True) + standardSAM=True, includeminute=True) From ebded8351eb468c02dbd0959188af5883bdc17a2 Mon Sep 17 00:00:00 2001 From: Silvana Ovaitt Date: Fri, 23 Sep 2022 14:19:31 -0600 Subject: [PATCH 09/20] Anton's suggestion --- pvlib/iotools/sam.py | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/pvlib/iotools/sam.py b/pvlib/iotools/sam.py index 638a5d4911..c4824012f1 100644 --- a/pvlib/iotools/sam.py +++ b/pvlib/iotools/sam.py @@ -109,24 +109,19 @@ def _fillYearSAMStyle(df, freq='60T'): savedata['Tdry'] = temp_amb if 'dni' in data: - dni = list(data.dni) - savedata['DHI'] = dni + savedata['DHI'] = data.dni.values if 'dhi' in data: - dhi = list(data.dhi) - savedata['DNI'] = dhi + savedata['DNI'] = data.dhi.values if 'ghi' in data: - ghi = list(data.ghi) - savedata['GHI'] = ghi + savedata['GHI'] = data.ghi.values if 'poa' in data: - poa = list(data.poa) - savedata['POA'] = poa + savedata['POA'] = data.poa.values if 'albedo' in data: - albedo = list(data.albedo) - savedata['Albedo'] = albedo + savedata['Albedo'] = data.albedo.values # Not elegant but seems to work for the standardSAM format if standardSAM and savedata.Albedo.iloc[0] == 0: From 82a24c9eba6d179d2ebeccb1d6b154ba0d41ad34 Mon Sep 17 00:00:00 2001 From: Silvana Ovaitt Date: Fri, 7 Oct 2022 11:18:10 +0200 Subject: [PATCH 10/20] update samp.py got it wrong, it should be left labeled. Added other values of use for SAM for wind simulations --- pvlib/iotools/sam.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/pvlib/iotools/sam.py b/pvlib/iotools/sam.py index c4824012f1..449af467f2 100644 --- a/pvlib/iotools/sam.py +++ b/pvlib/iotools/sam.py @@ -47,7 +47,7 @@ def _is_leap_and_29Feb(s): ((s.index.year % 100 != 0) | (s.index.year % 400 == 0)) & \ (s.index.month == 2) & (s.index.day == 29) - def _averageSAMStyle(df, interval='60T', closed='right', label='right'): + def _averageSAMStyle(df, interval='60T', closed='left', label='left'): ''' Averages subhourly data into hourly data in SAM's expected format. ''' df = df.resample(interval, closed=closed, label=label).mean() @@ -120,6 +120,15 @@ def _fillYearSAMStyle(df, freq='60T'): if 'poa' in data: savedata['POA'] = data.poa.values + if 'rh' in data: + savedata['rh'] = data.rh.values + + if 'pressure' in data: + savedata['pressure'] = data.pressure.values + + if 'wdir' in data: + savedata['wdir'] = data.wdir.values + if 'albedo' in data: savedata['Albedo'] = data.albedo.values From 9d3c87b150936fab8119cf4782527b1db96910df Mon Sep 17 00:00:00 2001 From: Silvana Ovaitt Date: Tue, 11 Oct 2022 10:49:24 -0600 Subject: [PATCH 11/20] pvlib.iotools.read_psm3 updates for SAM The function description states that it reads SAM files, but the SAM files does not need a 'Local Time Zone' in the metadata, as well as the Minute column is optional. When Minutes are included in the SAM file, it considers the sun position needs to be calculated at that specific point, instead of calculating it for left labeled data (timestamp 10 AM is data left averaged from 10 AM to 11 AM, and SAM places Sun position at 10:30 AM). --- pvlib/iotools/psm3.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/pvlib/iotools/psm3.py b/pvlib/iotools/psm3.py index 71e74dfd54..c0c3102291 100644 --- a/pvlib/iotools/psm3.py +++ b/pvlib/iotools/psm3.py @@ -314,7 +314,8 @@ def parse_psm3(fbuf, map_variables=None): metadata_values[-1] = metadata_values[-1].strip() # strip trailing newline metadata = dict(zip(metadata_fields, metadata_values)) # the response is all strings, so set some metadata types to numbers - metadata['Local Time Zone'] = int(metadata['Local Time Zone']) + if 'Local Time Zone' in metadata: + metadata['Local Time Zone'] = int(metadata['Local Time Zone']) metadata['Time Zone'] = int(metadata['Time Zone']) metadata['Latitude'] = float(metadata['Latitude']) metadata['Longitude'] = float(metadata['Longitude']) @@ -333,8 +334,13 @@ def parse_psm3(fbuf, map_variables=None): fbuf, header=None, names=columns, usecols=columns, dtype=dtypes, delimiter=',', lineterminator='\n') # skip carriage returns \r # the response 1st 5 columns are a date vector, convert to datetime - dtidx = pd.to_datetime( - data[['Year', 'Month', 'Day', 'Hour', 'Minute']]) + # SAM foramt only specifies minutes for specific solar position modeling. + if 'Minute' in data.columns: + dtidx = pd.to_datetime( + data[['Year', 'Month', 'Day', 'Hour', 'Minute']]) + else: + dtidx = pd.to_datetime( + data[['Year', 'Month', 'Day', 'Hour']]) # in USA all timezones are integers tz = 'Etc/GMT%+d' % -metadata['Time Zone'] data.index = pd.DatetimeIndex(dtidx).tz_localize(tz) From 5d7249109dff1fe082d63638c9ca7ba2d6fd6b56 Mon Sep 17 00:00:00 2001 From: Silvana Ovaitt Date: Thu, 13 Oct 2022 16:17:22 -0600 Subject: [PATCH 12/20] updates williamhobbs -metadata now uses and expects 'tz' (non caps). -If tz data is not numberic, it throws an error. -function renamed to write_sam() -source value set to 'pvlib export' if not defined. --- pvlib/iotools/psm3.py | 2 +- pvlib/iotools/sam.py | 28 +++++++++++++++++++--------- 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/pvlib/iotools/psm3.py b/pvlib/iotools/psm3.py index c0c3102291..29712f7470 100644 --- a/pvlib/iotools/psm3.py +++ b/pvlib/iotools/psm3.py @@ -338,7 +338,7 @@ def parse_psm3(fbuf, map_variables=None): if 'Minute' in data.columns: dtidx = pd.to_datetime( data[['Year', 'Month', 'Day', 'Hour', 'Minute']]) - else: + else: dtidx = pd.to_datetime( data[['Year', 'Month', 'Day', 'Hour']]) # in USA all timezones are integers diff --git a/pvlib/iotools/sam.py b/pvlib/iotools/sam.py index 449af467f2..2ff407692f 100644 --- a/pvlib/iotools/sam.py +++ b/pvlib/iotools/sam.py @@ -3,7 +3,7 @@ import pandas as pd -def saveSAM_WeatherFile(data, metadata, savefile='SAM_WeatherFile.csv', +def write_sam(data, metadata, savefile='SAM_WeatherFile.csv', standardSAM=True, includeminute=False): """ Saves dataframe with weather data from pvlib format on SAM-friendly format. @@ -11,12 +11,12 @@ def saveSAM_WeatherFile(data, metadata, savefile='SAM_WeatherFile.csv', Parameters ----------- data : pandas.DataFrame - timeseries data in PVLib format. Should be TZ converted (not UTC). + timeseries data in PVLib format. Should be tz converted (not UTC). Ideally it is one sequential year data; if not suggested to use standardSAM = False. metdata : dictionary Dictionary with 'latitude', 'longitude', 'elevation', 'source', - and 'TZ' for timezone. + and 'tz' for timezone. savefile : str Name of file to save output as. standardSAM : boolean @@ -58,7 +58,7 @@ def _fillYearSAMStyle(df, freq='60T'): ''' # add zeros for the rest of the year # add a timepoint at the end of the year - # apply correct TZ info (if applicable) + # apply correct tz info (if applicable) tzinfo = df.index.tzinfo starttime = pd.to_datetime('%s-%s-%s %s:%s' % (df.index.year[0], 1, 1, 0, 0) @@ -86,8 +86,13 @@ def _fillYearSAMStyle(df, freq='60T'): latitude = metadata['latitude'] longitude = metadata['longitude'] elevation = metadata['elevation'] - timezone_offset = metadata['TZ'] - source = metadata['source'] + timezone_offset = metadata['tz'] + + if 'source' in metadata: + source = metadata['source'] + else: + source = 'pvlib export' + metadata['source'] = source # make a header header = '\n'.join(['Source,Latitude,Longitude,Time Zone,Elevation', @@ -125,7 +130,7 @@ def _fillYearSAMStyle(df, freq='60T'): if 'pressure' in data: savedata['pressure'] = data.pressure.values - + if 'wdir' in data: savedata['wdir'] = data.wdir.values @@ -172,11 +177,16 @@ def tz_convert(df, tz_convert_val, metadata=None): Adds (or updates) the existing Timezone in the metadata dictionary """ + + if isinstance(tz_convert_val, int) is False: + print("Please pass a numeric timezone, i.e. -6 for MDT or 0 for UTC.") + return + import pytz if (type(tz_convert_val) == int) | (type(tz_convert_val) == float): df = df.tz_convert(pytz.FixedOffset(tz_convert_val*60)) - if metadata is not None: - metadata['TZ'] = tz_convert_val + if 'tz' in metadata: + metadata['tz'] = tz_convert_val return df, metadata return df From 82de598b4b17faffc98b0362d5605daba3b07c29 Mon Sep 17 00:00:00 2001 From: Silvana Ovaitt Date: Thu, 13 Oct 2022 16:21:40 -0600 Subject: [PATCH 13/20] propagate new write_sam name --- docs/sphinx/source/whatsnew/v0.9.4.rst | 2 +- pvlib/iotools/__init__.py | 2 +- pvlib/tests/iotools/test_sam.py | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/sphinx/source/whatsnew/v0.9.4.rst b/docs/sphinx/source/whatsnew/v0.9.4.rst index da1d29d58d..1daa7da4ba 100644 --- a/docs/sphinx/source/whatsnew/v0.9.4.rst +++ b/docs/sphinx/source/whatsnew/v0.9.4.rst @@ -9,7 +9,7 @@ Deprecations Enhancements ~~~~~~~~~~~~ -* :py:func:`pvlib.iotools.saveSAM_WeatherFile` now writes pvlib formated dataframe +* :py:func:`pvlib.iotools.write_sam` now writes pvlib formated dataframe into a weatherfile usable in SAM. (:pull:`1556`, :issue:`1371`) Bug fixes diff --git a/pvlib/iotools/__init__.py b/pvlib/iotools/__init__.py index 0b5231ac57..b5bf25e27b 100644 --- a/pvlib/iotools/__init__.py +++ b/pvlib/iotools/__init__.py @@ -21,4 +21,4 @@ from pvlib.iotools.sodapro import get_cams # noqa: F401 from pvlib.iotools.sodapro import read_cams # noqa: F401 from pvlib.iotools.sodapro import parse_cams # noqa: F401 -from pvlib.iotools.sam import saveSAM_WeatherFile, tz_convert # noqa: F401 +from pvlib.iotools.sam import write_sam, tz_convert # noqa: F401 diff --git a/pvlib/tests/iotools/test_sam.py b/pvlib/tests/iotools/test_sam.py index e92dc7bd30..b7632949c2 100644 --- a/pvlib/tests/iotools/test_sam.py +++ b/pvlib/tests/iotools/test_sam.py @@ -2,7 +2,7 @@ test the SAM IO tools """ from pvlib.iotools import get_pvgis_tmy -from pvlib.iotools import saveSAM_WeatherFile, tz_convert +from pvlib.iotools import write_sam, tz_convert from ..conftest import DATA_DIR import numpy as np @@ -26,5 +26,5 @@ def test_saveSAM_WeatherFile(): coerce_year = 2021 data['poa'] = np.nan data.index = data.index.map(lambda dt: dt.replace(year=coerce_year)) - saveSAM_WeatherFile(data, metadata, savefile='test_SAMWeatherFile.csv', + write_sam(data, metadata, savefile='test_SAMWeatherFile.csv', standardSAM=True, includeminute=True) From 747acc8238dae4efcdf969829ddd4588b5b4e86b Mon Sep 17 00:00:00 2001 From: Silvana Ovaitt Date: Thu, 13 Oct 2022 16:24:55 -0600 Subject: [PATCH 14/20] silently cursing the stickler in spanish.. --- pvlib/iotools/sam.py | 6 +++--- pvlib/tests/iotools/test_sam.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pvlib/iotools/sam.py b/pvlib/iotools/sam.py index 2ff407692f..ea40271596 100644 --- a/pvlib/iotools/sam.py +++ b/pvlib/iotools/sam.py @@ -4,7 +4,7 @@ def write_sam(data, metadata, savefile='SAM_WeatherFile.csv', - standardSAM=True, includeminute=False): + standardSAM=True, includeminute=False): """ Saves dataframe with weather data from pvlib format on SAM-friendly format. @@ -87,7 +87,7 @@ def _fillYearSAMStyle(df, freq='60T'): longitude = metadata['longitude'] elevation = metadata['elevation'] timezone_offset = metadata['tz'] - + if 'source' in metadata: source = metadata['source'] else: @@ -177,7 +177,7 @@ def tz_convert(df, tz_convert_val, metadata=None): Adds (or updates) the existing Timezone in the metadata dictionary """ - + if isinstance(tz_convert_val, int) is False: print("Please pass a numeric timezone, i.e. -6 for MDT or 0 for UTC.") return diff --git a/pvlib/tests/iotools/test_sam.py b/pvlib/tests/iotools/test_sam.py index b7632949c2..01bdb91699 100644 --- a/pvlib/tests/iotools/test_sam.py +++ b/pvlib/tests/iotools/test_sam.py @@ -27,4 +27,4 @@ def test_saveSAM_WeatherFile(): data['poa'] = np.nan data.index = data.index.map(lambda dt: dt.replace(year=coerce_year)) write_sam(data, metadata, savefile='test_SAMWeatherFile.csv', - standardSAM=True, includeminute=True) + standardSAM=True, includeminute=True) From 1b684e0aa4181e04b93906ffa88fe26f30a32309 Mon Sep 17 00:00:00 2001 From: Silvana Ovaitt Date: Thu, 13 Oct 2022 16:25:47 -0600 Subject: [PATCH 15/20] again --- pvlib/tests/iotools/test_sam.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pvlib/tests/iotools/test_sam.py b/pvlib/tests/iotools/test_sam.py index 01bdb91699..91e200057c 100644 --- a/pvlib/tests/iotools/test_sam.py +++ b/pvlib/tests/iotools/test_sam.py @@ -27,4 +27,4 @@ def test_saveSAM_WeatherFile(): data['poa'] = np.nan data.index = data.index.map(lambda dt: dt.replace(year=coerce_year)) write_sam(data, metadata, savefile='test_SAMWeatherFile.csv', - standardSAM=True, includeminute=True) + standardSAM=True, includeminute=True) From f366fcf0b71cb1763dd423b97b065b29c315f7da Mon Sep 17 00:00:00 2001 From: Silvana Ovaitt Date: Thu, 13 Oct 2022 16:26:38 -0600 Subject: [PATCH 16/20] i don't get it --- pvlib/tests/iotools/test_sam.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pvlib/tests/iotools/test_sam.py b/pvlib/tests/iotools/test_sam.py index 91e200057c..e52067b85e 100644 --- a/pvlib/tests/iotools/test_sam.py +++ b/pvlib/tests/iotools/test_sam.py @@ -27,4 +27,4 @@ def test_saveSAM_WeatherFile(): data['poa'] = np.nan data.index = data.index.map(lambda dt: dt.replace(year=coerce_year)) write_sam(data, metadata, savefile='test_SAMWeatherFile.csv', - standardSAM=True, includeminute=True) + standardSAM=True, includeminute=True) From 0dfa8470a3ef247a2dc1525f1eef3b2cd23321f4 Mon Sep 17 00:00:00 2001 From: Silvana Ovaitt Date: Fri, 14 Oct 2022 16:18:23 -0600 Subject: [PATCH 17/20] Update pvlib/tests/iotools/test_sam.py Co-authored-by: Kevin Anderson --- pvlib/tests/iotools/test_sam.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pvlib/tests/iotools/test_sam.py b/pvlib/tests/iotools/test_sam.py index e52067b85e..de582ce364 100644 --- a/pvlib/tests/iotools/test_sam.py +++ b/pvlib/tests/iotools/test_sam.py @@ -27,4 +27,4 @@ def test_saveSAM_WeatherFile(): data['poa'] = np.nan data.index = data.index.map(lambda dt: dt.replace(year=coerce_year)) write_sam(data, metadata, savefile='test_SAMWeatherFile.csv', - standardSAM=True, includeminute=True) + standardSAM=True, includeminute=True) From 492e3738e80032bd6dc9362e6dda314e4b373038 Mon Sep 17 00:00:00 2001 From: Silvana Ovaitt Date: Fri, 14 Oct 2022 16:59:36 -0600 Subject: [PATCH 18/20] loop data addresses data looping when data goes from example June one year to May the next one. --- pvlib/iotools/sam.py | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/pvlib/iotools/sam.py b/pvlib/iotools/sam.py index ea40271596..80225ce331 100644 --- a/pvlib/iotools/sam.py +++ b/pvlib/iotools/sam.py @@ -67,20 +67,19 @@ def _fillYearSAMStyle(df, freq='60T'): 23, 60-int(freq[:-1])) ).tz_localize(tzinfo) - df2 = _averageSAMStyle(df, freq) - df2.iloc[0] = 0 # set first datapt to zero to forward fill w zeros - df2.iloc[-1] = 0 # set last datapt to zero to forward fill w zeros - df2.loc[starttime] = 0 - df2.loc[endtime] = 0 - df2 = df2.resample(freq).ffill() - return df2 + df.iloc[0] = 0 # set first datapt to zero to forward fill w zeros + df.iloc[-1] = 0 # set last datapt to zero to forward fill w zeros + df.loc[starttime] = 0 + df.loc[endtime] = 0 + df = df.resample(freq).ffill() + return df # Modify this to cut into different years. Right now handles partial year # and sub-hourly interval. if standardSAM: + data = _averageSAMStyle(data, '60T') filterdatesLeapYear = ~(_is_leap_and_29Feb(data)) data = data[filterdatesLeapYear] - data = _fillYearSAMStyle(data) # metadata latitude = metadata['latitude'] @@ -134,15 +133,23 @@ def _fillYearSAMStyle(df, freq='60T'): if 'wdir' in data: savedata['wdir'] = data.wdir.values + savedata = savedata.sort_values(by=['Month','Day','Hour']) + if 'albedo' in data: savedata['Albedo'] = data.albedo.values - # Not elegant but seems to work for the standardSAM format + if standardSAM and len(data) < 8760: + savedata = _fillYearSAMStyle(savedata) + + # Not elegant but seems to work for the standardSAM format + if 'Albedo' in savedata: if standardSAM and savedata.Albedo.iloc[0] == 0: savedata.loc[savedata.index[0], 'Albedo'] = savedata.loc[ savedata.index[1]]['Albedo'] savedata.loc[savedata.index[-1], 'Albedo'] = savedata.loc[ savedata.index[-2]]['Albedo'] + savedata['Albedo'] = savedata['Albedo'].fillna(0.99).clip(lower=0.01, + upper=0.99) with open(savefile, 'w', newline='') as ict: # Write the header lines, including the index variable for From e4356f98307910b8aa3335aeba1a5ca5e805527e Mon Sep 17 00:00:00 2001 From: Silvana Ovaitt Date: Fri, 14 Oct 2022 17:00:41 -0600 Subject: [PATCH 19/20] asdf --- pvlib/iotools/sam.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pvlib/iotools/sam.py b/pvlib/iotools/sam.py index 80225ce331..4369cda949 100644 --- a/pvlib/iotools/sam.py +++ b/pvlib/iotools/sam.py @@ -133,7 +133,7 @@ def _fillYearSAMStyle(df, freq='60T'): if 'wdir' in data: savedata['wdir'] = data.wdir.values - savedata = savedata.sort_values(by=['Month','Day','Hour']) + savedata = savedata.sort_values(by=['Month', 'Day', 'Hour']) if 'albedo' in data: savedata['Albedo'] = data.albedo.values @@ -149,7 +149,7 @@ def _fillYearSAMStyle(df, freq='60T'): savedata.loc[savedata.index[-1], 'Albedo'] = savedata.loc[ savedata.index[-2]]['Albedo'] savedata['Albedo'] = savedata['Albedo'].fillna(0.99).clip(lower=0.01, - upper=0.99) + upper=0.99) with open(savefile, 'w', newline='') as ict: # Write the header lines, including the index variable for From 34f2f9cdd17928ba1eceb7031aae50411c759dee Mon Sep 17 00:00:00 2001 From: Silvana Ovaitt Date: Mon, 10 Apr 2023 05:56:21 -0600 Subject: [PATCH 20/20] DNI and DHI typo --- pvlib/iotools/sam.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pvlib/iotools/sam.py b/pvlib/iotools/sam.py index 4369cda949..254f21a188 100644 --- a/pvlib/iotools/sam.py +++ b/pvlib/iotools/sam.py @@ -113,10 +113,10 @@ def _fillYearSAMStyle(df, freq='60T'): savedata['Tdry'] = temp_amb if 'dni' in data: - savedata['DHI'] = data.dni.values + savedata['DNI'] = data.dni.values if 'dhi' in data: - savedata['DNI'] = data.dhi.values + savedata['DHI'] = data.dhi.values if 'ghi' in data: savedata['GHI'] = data.ghi.values