Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions benchmarks/benchmarks.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
from hipscat.catalog import Catalog, PartitionInfo
from hipscat.catalog.association_catalog.partition_join_info import PartitionJoinInfo
from hipscat.catalog.catalog_info import CatalogInfo
from hipscat.io.paths import pixel_catalog_files
from hipscat.pixel_math import HealpixPixel
from hipscat.pixel_tree import PixelAlignment, align_trees
from hipscat.pixel_tree.pixel_tree_builder import PixelTreeBuilder
Expand Down Expand Up @@ -55,6 +56,9 @@ def setup(self):
def time_pixel_tree_creation(self):
PixelTreeBuilder.from_healpix(self.pixel_list)

def time_paths_creation(self):
pixel_catalog_files("foo/", self.pixel_list)


class MetadataSuite:
"""Suite that generates catalog files and benchmarks the operations on them."""
Expand Down
39 changes: 38 additions & 1 deletion src/hipscat/io/paths.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,9 @@
from __future__ import annotations

import re
from typing import Dict, List

from hipscat.io.file_io.file_pointer import FilePointer, append_paths_to_pointer
from hipscat.io.file_io.file_pointer import FilePointer, append_paths_to_pointer, get_fs
from hipscat.pixel_math.healpix_pixel import INVALID_PIXEL, HealpixPixel

ORDER_DIRECTORY_PREFIX = "Norder"
Expand Down Expand Up @@ -85,6 +86,42 @@ def get_healpix_from_path(path: str) -> HealpixPixel:
return HealpixPixel(int(order), int(pixel))


def pixel_catalog_files(
catalog_base_dir: FilePointer, pixels: List[HealpixPixel], storage_options: Dict | None = None
) -> List[FilePointer]:
"""Create a list of path *pointers* for pixel catalog files. This will not create the directory
or files.

The catalog file names will take the HiPS standard form of::

<catalog_base_dir>/Norder=<pixel_order>/Dir=<directory number>/Npix=<pixel_number>.parquet

Where the directory number is calculated using integer division as::

(pixel_number/10000)*10000

Args:
catalog_base_dir (FilePointer): base directory of the catalog (includes catalog name)
pixels (List[HealpixPixel]): the healpix pixels to create pointers to
storage_options (dict): the storage options for the file system to target when generating the paths
Returns (List[FilePointer]):
A list of paths to the pixels, in the same order as the input pixel list.
"""
fs, _ = get_fs(catalog_base_dir, storage_options)
base_path_stripped = catalog_base_dir.removesuffix(fs.sep)
return [
fs.sep.join(
[
base_path_stripped,
f"{ORDER_DIRECTORY_PREFIX}={pixel.order}",
f"{DIR_DIRECTORY_PREFIX}={pixel.dir}",
f"{PIXEL_DIRECTORY_PREFIX}={pixel.pixel}.parquet",
]
)
for pixel in pixels
]


def pixel_catalog_file(catalog_base_dir: FilePointer, pixel_order: int, pixel_number: int) -> FilePointer:
"""Create path *pointer* for a pixel catalog file. This will not create the directory
or file.
Expand Down
6 changes: 6 additions & 0 deletions tests/hipscat/io/test_paths.py
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,12 @@ def test_pixel_catalog_file_nonint():
paths.pixel_catalog_file("/foo", "zero", "five")


def test_pixel_catalog_files():
expected = ["/foo/Norder=0/Dir=0/Npix=5.parquet", "/foo/Norder=1/Dir=0/Npix=16.parquet"]
result = paths.pixel_catalog_files("/foo", [HealpixPixel(0, 5), HealpixPixel(1, 16)])
assert expected == result


def test_get_healpix_from_path():
expected = HealpixPixel(5, 34)

Expand Down