|
| 1 | +import contextlib |
| 2 | +import os |
| 3 | +from pathlib import Path |
| 4 | +from typing import Union |
| 5 | + |
| 6 | +from astropy.table import Table, hstack |
| 7 | + |
| 8 | +import fibad.downloadCutout.downloadCutout as dC |
| 9 | + |
| 10 | +# These are the fields that are allowed to vary across the locations |
| 11 | +# input from the catalog fits file. Other values for HSC cutout server |
| 12 | +# must be provided by config. |
| 13 | +# |
| 14 | +# Order here is intentional, this is also a sort order to optimize |
| 15 | +# queries to the cutout server. |
| 16 | +variable_fields = ["tract", "ra", "dec"] |
| 17 | + |
| 18 | + |
| 19 | +@contextlib.contextmanager |
| 20 | +def working_directory(path: Path): |
| 21 | + """ |
| 22 | + Context Manager to change our working directory. |
| 23 | +
|
| 24 | + Supports downloadCutouts which always writes to cwd. |
| 25 | + """ |
| 26 | + old_cwd = Path.cwd() |
| 27 | + os.chdir(path) |
| 28 | + try: |
| 29 | + yield |
| 30 | + finally: |
| 31 | + os.chdir(old_cwd) |
| 32 | + |
| 33 | + |
| 34 | +def run(args, config): |
| 35 | + """ |
| 36 | + Main entrypoint for downloading cutouts from HSC for use with fibad |
| 37 | + """ |
| 38 | + |
| 39 | + config = config.get("download", {}) |
| 40 | + |
| 41 | + print("Download command") |
| 42 | + |
| 43 | + # Filter the fits file for the fields we want |
| 44 | + column_names = ["object_id"] + variable_fields |
| 45 | + locations = filterfits(config.get("fits_file"), column_names) |
| 46 | + |
| 47 | + # Sort by tract, ra, dec to optimize speed that the cutout server can serve us |
| 48 | + # |
| 49 | + # TODO: See if this sort is performed by downloadCutouts |
| 50 | + # It appears downloadCutouts is doing some sorting prior to download, but |
| 51 | + # unclear if it is the same sort |
| 52 | + locations.sort(variable_fields) |
| 53 | + |
| 54 | + # TODO slice up the locations |
| 55 | + locations = locations[0:10] |
| 56 | + |
| 57 | + # make a list of rects |
| 58 | + rects = create_rects(locations, offset=0, default=rect_from_config(config)) |
| 59 | + |
| 60 | + # Configure global parameters for the downloader |
| 61 | + dC.set_max_connections(num=config.get("max_connections", 2)) |
| 62 | + |
| 63 | + # pass the rects to the cutout downloader |
| 64 | + download_cutout_group( |
| 65 | + rects=rects, cutout_dir=config.get("cutout_dir"), user=config["username"], password=config["password"] |
| 66 | + ) |
| 67 | + |
| 68 | + print(locations) |
| 69 | + |
| 70 | + |
| 71 | +# TODO add error checking |
| 72 | +def filterfits(filename: str, column_names: list[str]) -> Table: |
| 73 | + """ |
| 74 | + Read a fits file with the required column names for making cutouts |
| 75 | +
|
| 76 | + Returns an astropy table containing only the necessary fields |
| 77 | +
|
| 78 | + The easiest way to make one of these is to select from the main HSC catalog |
| 79 | + """ |
| 80 | + t = Table.read(filename) |
| 81 | + columns = [t[column] for column in column_names] |
| 82 | + return hstack(columns, uniq_col_name="{table_name}", table_names=column_names) |
| 83 | + |
| 84 | + |
| 85 | +def rect_from_config(config: dict) -> dC.Rect: |
| 86 | + """ |
| 87 | + Takes our Download config and loads cutout config |
| 88 | + common to all cutouts into a prototypical Rect for downloading |
| 89 | + """ |
| 90 | + return dC.Rect.create( |
| 91 | + sw=config["sw"], |
| 92 | + sh=config["sh"], |
| 93 | + filter=config["filter"], |
| 94 | + rerun=config["rerun"], |
| 95 | + type=config["type"], |
| 96 | + ) |
| 97 | + |
| 98 | + |
| 99 | +def create_rects(locations: Table, offset: int = 0, default: dC.Rect = None) -> list[dC.Rect]: |
| 100 | + """ |
| 101 | + Create the rects we will need to pass to the downloader. |
| 102 | + One Rect per location in our list of sky locations. |
| 103 | +
|
| 104 | + Rects are created with all fields in the default rect pre-filled |
| 105 | +
|
| 106 | + Offset here is to allow multiple downloads on different sections of the source list |
| 107 | + without file name clobbering during the download phase. The offset is intended to be |
| 108 | + the index of the start of the locations table within some larger fits file. |
| 109 | + """ |
| 110 | + rects = [] |
| 111 | + for index, location in enumerate(locations): |
| 112 | + args = {field: location[field] for field in variable_fields} |
| 113 | + args["lineno"] = index + offset |
| 114 | + args["tract"] = str(args["tract"]) |
| 115 | + rect = dC.Rect.create(default=default, **args) |
| 116 | + rects.append(rect) |
| 117 | + |
| 118 | + return rects |
| 119 | + |
| 120 | + |
| 121 | +def download_cutout_group(rects: list[dC.Rect], cutout_dir: Union[str, Path], user, password): |
| 122 | + """ |
| 123 | + Download cutouts to the given directory |
| 124 | +
|
| 125 | + Calls downloadCutout.download, so supports long lists of rects and |
| 126 | + """ |
| 127 | + with working_directory(Path(cutout_dir)): |
| 128 | + dC.download(rects, user=user, password=password, onmemory=False) |
0 commit comments