Clean up code related to multiple download attempts

[williamhobbs]

There are many places where Herbie or FastHerbie could be called with each download attempt. This should be moved outside the 'attempts' loop to simplify code and (maybe) speed things up a bit.

Here's an example where it was done as I'm proposing here:

hefty/src/hefty/solar.py

Lines 487 to 507 in ff01537

	FH = FastHerbie([date], model=model, product=product, fxx=fxx_range,
	member=member, priority=priority)
	for j in range(0, len(search_string_list)):
	# get solar, 10m wind, and 2m temp data
	# try n times based loosely on
	# https://thingspython.wordpress.com/2021/12/05/how-to-try-something-n-times-in-python/
	for attempts_remaining in reversed(range(attempts)):
	attempt_num = attempts - attempts_remaining
	try:
	if attempt_num == 1:
	# try downloading
	FH.download(search_string_list[j])
	ds_dict[j] = FH.xarray(search_string_list[j],
	remove_grib=True)
	else:
	# after first attempt, set overwrite=True to overwrite
	# partial files
	FH.download(search_string_list[j])
	ds_dict[j] = FH.xarray(search_string_list[j],
	remove_grib=True,
	overwrite=True)

And examples of where it needs to be changed:

hefty/src/hefty/solar.py

Lines 106 to 129 in ff01537

	for attempts_remaining in reversed(range(attempts)):
	attempt_num = attempts - attempts_remaining
	try:
	if attempt_num == 1:
	# try downloading
	ds = Herbie(
	date,
	model=model,
	product=product,
	fxx=fxx,
	member=member,
	priority=priority
	).xarray(search_str)
	else:
	# after first attempt, set overwrite=True to overwrite
	# partial files
	ds = Herbie(
	date,
	model=model,
	product=product,
	fxx=fxx,
	member=member,
	priority=priority
	).xarray(search_str, overwrite=True)

hefty/src/hefty/solar.py

Lines 867 to 885 in ff01537

	for attempts_remaining in reversed(range(attempts)):
	attempt_num = attempts - attempts_remaining
	try:
	if attempt_num == 1:
	# try downloading
	ds = FastHerbie(DATES=[init_date],
	model='ifs',
	product='enfo',
	fxx=fxx_range,
	priority=priority).xarray(search_str)
	else:
	# after first attempt, set overwrite=True to overwrite
	# partial files
	ds = FastHerbie(DATES=[init_date],
	model='ifs',
	product='enfo',
	fxx=fxx_range,
	priority=priority).xarray(search_str,
	overwrite=True)

hefty/src/hefty/solar.py

Lines 1021 to 1037 in ff01537

	for attempts_remaining in reversed(range(attempts)):
	attempt_num = attempts - attempts_remaining
	try:
	if attempt_num == 1:
	# try downloading
	ds = FastHerbie(DATES=[init_date],
	model='ifs',
	product='enfo',
	fxx=fxx_range).xarray(search_str)
	else:
	# after first attempt, set overwrite=True to overwrite
	# partial files
	ds = FastHerbie(DATES=[init_date],
	model='ifs',
	product='enfo',
	fxx=fxx_range).xarray(search_str,
	overwrite=True)

hefty/src/hefty/solar.py

Lines 1216 to 1247 in ff01537

	for attempts_remaining in reversed(range(attempts)):
	attempt_num = attempts - attempts_remaining
	try:
	if attempt_num == 1:
	# try downloading
	FH = FastHerbie(DATES=[init_date],
	model=model,
	product='enfo',
	fxx=fxx_range,
	priority=priority)
	FH.download(search_str)
	ds = FH.xarray(search_str, remove_grib=False)
	else:
	# after first attempt, set overwrite=True to overwrite
	# partial files
	# try downloading
	FH = FastHerbie(DATES=[init_date],
	model=model,
	product='enfo',
	fxx=fxx_range,
	priority=priority)
	FH.download(search_str, overwrite=True)
	ds = FH.xarray(search_str, remove_grib=False)
	except Exception:
	if attempts_remaining:
	print('attempt ' + str(attempt_num) + ' failed, pause for '
	+ str((attempt_num)**2) + ' min')
	time.sleep(60*(attempt_num)**2)
	else:
	break
	else:
	raise ValueError('download failed, ran out of attempts')

hefty/src/hefty/solar.py

Lines 1670 to 1700 in ff01537

	for attempts_remaining in reversed(range(attempts)):
	attempt_num = attempts - attempts_remaining
	try:
	if attempt_num == 1:
	# try downloading
	FH = FastHerbie(DATES=[init_date],
	model=model,
	product=product,
	fxx=fxx_range,
	member=member)
	FH.download(search_str)
	ds = FH.xarray(search_str, remove_grib=False)
	else:
	# after first attempt, set overwrite=True to overwrite
	# partial files
	FH = FastHerbie(DATES=[init_date],
	model=model,
	product=product,
	fxx=fxx_range,
	member=member)
	FH.download(search_str, overwrite=True)
	ds = FH.xarray(search_str, remove_grib=False)
	except Exception:
	if attempts_remaining:
	print('attempt ' + str(attempt_num) + ' failed, pause for '
	+ str((attempt_num)**2) + ' min')
	time.sleep(60*(attempt_num)**2)
	else:
	break
	else:
	raise ValueError('download failed, ran out of attempts')

And in wind.py and custom.py:

hefty/src/hefty/wind.py

Lines 104 to 127 in ff01537

	for attempts_remaining in reversed(range(attempts)):
	attempt_num = attempts - attempts_remaining
	try:
	if attempt_num == 1:
	# try downloading
	ds = Herbie(
	date,
	model=model,
	product=product,
	fxx=fxx,
	member=member,
	priority=priority
	).xarray(search_str)
	else:
	# after first attempt, set overwrite=True to overwrite
	# partial files
	ds = Herbie(
	date,
	model=model,
	product=product,
	fxx=fxx,
	member=member,
	priority=priority
	).xarray(search_str, overwrite=True)

hefty/src/hefty/custom.py

Lines 104 to 127 in ff01537

	for attempts_remaining in reversed(range(attempts)):
	attempt_num = attempts - attempts_remaining
	try:
	if attempt_num == 1:
	# try downloading
	ds = Herbie(
	date,
	model=model,
	product=product,
	fxx=fxx,
	member=member,
	priority=priority
	).xarray(search_str)
	else:
	# after first attempt, set overwrite=True to overwrite
	# partial files
	ds = Herbie(
	date,
	model=model,
	product=product,
	fxx=fxx,
	member=member,
	priority=priority
	).xarray(search_str, overwrite=True)

[williamhobbs]

On second thought, if the error comes from not making the initial connection to the source, that would not be handled well outside the "attempts" loop. I'm now leaning towards not making this change. In which case, the example below should be swapped to match the others:

hefty/src/hefty/solar.py

Lines 487 to 507 in ff01537

	FH = FastHerbie([date], model=model, product=product, fxx=fxx_range,
	member=member, priority=priority)
	for j in range(0, len(search_string_list)):
	# get solar, 10m wind, and 2m temp data
	# try n times based loosely on
	# https://thingspython.wordpress.com/2021/12/05/how-to-try-something-n-times-in-python/
	for attempts_remaining in reversed(range(attempts)):
	attempt_num = attempts - attempts_remaining
	try:
	if attempt_num == 1:
	# try downloading
	FH.download(search_string_list[j])
	ds_dict[j] = FH.xarray(search_string_list[j],
	remove_grib=True)
	else:
	# after first attempt, set overwrite=True to overwrite
	# partial files
	FH.download(search_string_list[j])
	ds_dict[j] = FH.xarray(search_string_list[j],
	remove_grib=True,
	overwrite=True)