diff --git a/docs/sphinx/source/reference/iotools.rst b/docs/sphinx/source/reference/iotools.rst index 514aeac2f5..5af92429f5 100644 --- a/docs/sphinx/source/reference/iotools.rst +++ b/docs/sphinx/source/reference/iotools.rst @@ -37,6 +37,8 @@ of sources and file formats relevant to solar energy modeling. iotools.get_cams iotools.read_cams iotools.parse_cams + iotools.saveSAM_WeatherFile + iotools.tz_convert A :py:class:`~pvlib.location.Location` object may be created from metadata in some files. diff --git a/docs/sphinx/source/whatsnew/v0.9.4.rst b/docs/sphinx/source/whatsnew/v0.9.4.rst index ea89248743..1daa7da4ba 100644 --- a/docs/sphinx/source/whatsnew/v0.9.4.rst +++ b/docs/sphinx/source/whatsnew/v0.9.4.rst @@ -9,7 +9,8 @@ Deprecations Enhancements ~~~~~~~~~~~~ - +* :py:func:`pvlib.iotools.write_sam` now writes pvlib formated dataframe +into a weatherfile usable in SAM. (:pull:`1556`, :issue:`1371`) Bug fixes ~~~~~~~~~ @@ -33,3 +34,4 @@ Requirements Contributors ~~~~~~~~~~~~ +* Silvana Ovaitt (:ghuser:`shirubana`) \ No newline at end of file diff --git a/pvlib/iotools/__init__.py b/pvlib/iotools/__init__.py index b02ce243ae..b5bf25e27b 100644 --- a/pvlib/iotools/__init__.py +++ b/pvlib/iotools/__init__.py @@ -21,3 +21,4 @@ from pvlib.iotools.sodapro import get_cams # noqa: F401 from pvlib.iotools.sodapro import read_cams # noqa: F401 from pvlib.iotools.sodapro import parse_cams # noqa: F401 +from pvlib.iotools.sam import write_sam, tz_convert # noqa: F401 diff --git a/pvlib/iotools/psm3.py b/pvlib/iotools/psm3.py index 71e74dfd54..29712f7470 100644 --- a/pvlib/iotools/psm3.py +++ b/pvlib/iotools/psm3.py @@ -314,7 +314,8 @@ def parse_psm3(fbuf, map_variables=None): metadata_values[-1] = metadata_values[-1].strip() # strip trailing newline metadata = dict(zip(metadata_fields, metadata_values)) # the response is all strings, so set some metadata types to numbers - metadata['Local Time Zone'] = int(metadata['Local Time Zone']) + if 'Local Time Zone' in metadata: + metadata['Local Time Zone'] = int(metadata['Local Time Zone']) metadata['Time Zone'] = int(metadata['Time Zone']) metadata['Latitude'] = float(metadata['Latitude']) metadata['Longitude'] = float(metadata['Longitude']) @@ -333,8 +334,13 @@ def parse_psm3(fbuf, map_variables=None): fbuf, header=None, names=columns, usecols=columns, dtype=dtypes, delimiter=',', lineterminator='\n') # skip carriage returns \r # the response 1st 5 columns are a date vector, convert to datetime - dtidx = pd.to_datetime( - data[['Year', 'Month', 'Day', 'Hour', 'Minute']]) + # SAM foramt only specifies minutes for specific solar position modeling. + if 'Minute' in data.columns: + dtidx = pd.to_datetime( + data[['Year', 'Month', 'Day', 'Hour', 'Minute']]) + else: + dtidx = pd.to_datetime( + data[['Year', 'Month', 'Day', 'Hour']]) # in USA all timezones are integers tz = 'Etc/GMT%+d' % -metadata['Time Zone'] data.index = pd.DatetimeIndex(dtidx).tz_localize(tz) diff --git a/pvlib/iotools/sam.py b/pvlib/iotools/sam.py new file mode 100644 index 0000000000..254f21a188 --- /dev/null +++ b/pvlib/iotools/sam.py @@ -0,0 +1,199 @@ +"""Functions for reading and writing SAM data files.""" + +import pandas as pd + + +def write_sam(data, metadata, savefile='SAM_WeatherFile.csv', + standardSAM=True, includeminute=False): + """ + Saves dataframe with weather data from pvlib format on SAM-friendly format. + + Parameters + ----------- + data : pandas.DataFrame + timeseries data in PVLib format. Should be tz converted (not UTC). + Ideally it is one sequential year data; if not suggested to use + standardSAM = False. + metdata : dictionary + Dictionary with 'latitude', 'longitude', 'elevation', 'source', + and 'tz' for timezone. + savefile : str + Name of file to save output as. + standardSAM : boolean + This checks the dataframe to avoid having a leap day, then averages it + to SAM style (closed to the right), + and fills the years so it starst on YEAR/1/1 0:0 and ends on + YEAR/12/31 23:00. + includeminute ; Bool + For hourly data, if SAM input does not have Minutes, it calculates the + sun position 30 minutes prior to the hour (i.e. 12 timestamp means sun + position at 11:30). + If minutes are included, it will calculate the sun position at the time + of the timestamp (12:00 at 12:00) + Set to true if resolution of data is sub-hourly. + + Returns + ------- + Nothing, it just writes the file. + + """ + + def _is_leap_and_29Feb(s): + ''' Creates a mask to help remove Leap Years. Obtained from: + https://stackoverflow.com/questions/34966422/remove-leap-year-day- + from-pandas-dataframe/34966636 + ''' + return (s.index.year % 4 == 0) & \ + ((s.index.year % 100 != 0) | (s.index.year % 400 == 0)) & \ + (s.index.month == 2) & (s.index.day == 29) + + def _averageSAMStyle(df, interval='60T', closed='left', label='left'): + ''' Averages subhourly data into hourly data in SAM's expected format. + ''' + df = df.resample(interval, closed=closed, label=label).mean() + return df + + def _fillYearSAMStyle(df, freq='60T'): + ''' Fills year + ''' + # add zeros for the rest of the year + # add a timepoint at the end of the year + # apply correct tz info (if applicable) + tzinfo = df.index.tzinfo + starttime = pd.to_datetime('%s-%s-%s %s:%s' % (df.index.year[0], 1, 1, + 0, 0) + ).tz_localize(tzinfo) + endtime = pd.to_datetime('%s-%s-%s %s:%s' % (df.index.year[-1], 12, 31, + 23, 60-int(freq[:-1])) + ).tz_localize(tzinfo) + + df.iloc[0] = 0 # set first datapt to zero to forward fill w zeros + df.iloc[-1] = 0 # set last datapt to zero to forward fill w zeros + df.loc[starttime] = 0 + df.loc[endtime] = 0 + df = df.resample(freq).ffill() + return df + + # Modify this to cut into different years. Right now handles partial year + # and sub-hourly interval. + if standardSAM: + data = _averageSAMStyle(data, '60T') + filterdatesLeapYear = ~(_is_leap_and_29Feb(data)) + data = data[filterdatesLeapYear] + + # metadata + latitude = metadata['latitude'] + longitude = metadata['longitude'] + elevation = metadata['elevation'] + timezone_offset = metadata['tz'] + + if 'source' in metadata: + source = metadata['source'] + else: + source = 'pvlib export' + metadata['source'] = source + + # make a header + header = '\n'.join(['Source,Latitude,Longitude,Time Zone,Elevation', + source + ',' + str(latitude) + ',' + str(longitude) + + ',' + str(timezone_offset) + ',' + + str(elevation)])+'\n' + + savedata = pd.DataFrame({'Year': data.index.year, + 'Month': data.index.month, + 'Day': data.index.day, + 'Hour': data.index.hour}) + + if includeminute: + savedata['Minute'] = data.index.minute + + windspeed = list(data.wind_speed) + temp_amb = list(data.temp_air) + savedata['Wspd'] = windspeed + savedata['Tdry'] = temp_amb + + if 'dni' in data: + savedata['DNI'] = data.dni.values + + if 'dhi' in data: + savedata['DHI'] = data.dhi.values + + if 'ghi' in data: + savedata['GHI'] = data.ghi.values + + if 'poa' in data: + savedata['POA'] = data.poa.values + + if 'rh' in data: + savedata['rh'] = data.rh.values + + if 'pressure' in data: + savedata['pressure'] = data.pressure.values + + if 'wdir' in data: + savedata['wdir'] = data.wdir.values + + savedata = savedata.sort_values(by=['Month', 'Day', 'Hour']) + + if 'albedo' in data: + savedata['Albedo'] = data.albedo.values + + if standardSAM and len(data) < 8760: + savedata = _fillYearSAMStyle(savedata) + + # Not elegant but seems to work for the standardSAM format + if 'Albedo' in savedata: + if standardSAM and savedata.Albedo.iloc[0] == 0: + savedata.loc[savedata.index[0], 'Albedo'] = savedata.loc[ + savedata.index[1]]['Albedo'] + savedata.loc[savedata.index[-1], 'Albedo'] = savedata.loc[ + savedata.index[-2]]['Albedo'] + savedata['Albedo'] = savedata['Albedo'].fillna(0.99).clip(lower=0.01, + upper=0.99) + + with open(savefile, 'w', newline='') as ict: + # Write the header lines, including the index variable for + # the last one if you're letting Pandas produce that for you. + # (see above). + for line in header: + ict.write(line) + + savedata.to_csv(ict, index=False) + + +def tz_convert(df, tz_convert_val, metadata=None): + """ + Support function to convert metdata to a different local timezone. + Particularly for GIS weather files which are returned in UTC by default. + + Parameters + ---------- + df : DataFrame + A dataframe in UTC timezone + tz_convert_val : int + Convert timezone to this fixed value, following ISO standard + (negative values indicating West of UTC.) + Returns: metdata, metadata + + + Returns + ------- + df : DataFrame + Dataframe in the converted local timezone. + metadata : dict + Adds (or updates) the existing Timezone in the metadata dictionary + + """ + + if isinstance(tz_convert_val, int) is False: + print("Please pass a numeric timezone, i.e. -6 for MDT or 0 for UTC.") + return + + import pytz + if (type(tz_convert_val) == int) | (type(tz_convert_val) == float): + df = df.tz_convert(pytz.FixedOffset(tz_convert_val*60)) + + if 'tz' in metadata: + metadata['tz'] = tz_convert_val + return df, metadata + return df diff --git a/pvlib/tests/iotools/test_sam.py b/pvlib/tests/iotools/test_sam.py new file mode 100644 index 0000000000..de582ce364 --- /dev/null +++ b/pvlib/tests/iotools/test_sam.py @@ -0,0 +1,30 @@ +""" +test the SAM IO tools +""" +from pvlib.iotools import get_pvgis_tmy +from pvlib.iotools import write_sam, tz_convert +from ..conftest import DATA_DIR +import numpy as np + +# PVGIS Hourly tests +# The test files are actual files from PVGIS where the data section have been +# reduced to only a few lines +testfile_radiation_csv = DATA_DIR / \ + 'pvgis_hourly_Timeseries_45.000_8.000_SA_30deg_0deg_2016_2016.csv' + + +def test_saveSAM_WeatherFile(): + data, months_selected, inputs, metadata = get_pvgis_tmy(latitude=33, + longitude=-110, + map_variables=True) + metadata = {'latitude': inputs['location']['latitude'], + 'longitude': inputs['location']['longitude'], + 'elevation': inputs['location']['elevation'], + 'source': 'User-generated'} + data, metadata = tz_convert(data, tz_convert_val=-7, metadata=metadata) + data['albedo'] = 0.2 + coerce_year = 2021 + data['poa'] = np.nan + data.index = data.index.map(lambda dt: dt.replace(year=coerce_year)) + write_sam(data, metadata, savefile='test_SAMWeatherFile.csv', + standardSAM=True, includeminute=True)