Removes references to "table.sub_table" configuration settings when inappropriate (#405)

* Removes references to "table.sub_table" configuration settings where they are not needed.

* Update benchmarks/vector_db_benchmarks.py

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* Fixing another reference to the old double quote subtable.

* Updating subtables from `hyrax_cnn` to `HyraxCNN` and `random_dataset` to `HyraxRandomDataset`.

---------

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

Author: drewoldag

benchmarks/vector_db_benchmarks.py

@@ -28,12 +28,12 @@ def setup(self, vector_length, vector_db_implementation):
         self.h.config["model"]["name"] = "HyraxLoopback"
 
         # Default inference batch size is 512, so this should result in 4 batch files
-        self.h.config["data_set.random_dataset"]["size"] = 2048
-        self.h.config["data_set.random_dataset"]["seed"] = 0
-        self.h.config["data_set.random_dataset"]["shape"] = [vector_length]
+        self.h.config["data_set"]["HyraxRandomDataset"]["size"] = 2048
+        self.h.config["data_set"]["HyraxRandomDataset"]["seed"] = 0
+        self.h.config["data_set"]["HyraxRandomDataset"]["shape"] = [vector_length]
 
         # Qdrant requires the vector size in order to create its collections
-        self.h.config["vector_db.qdrant"]["vector_size"] = vector_length
+        self.h.config["vector_db"]["qdrant"]["vector_size"] = vector_length
 
         weights_file = self.input_dir / "fakeweights"
         with open(weights_file, "a"):
@@ -88,9 +88,9 @@ def setup(self, shard_size_limit, vector_db_implementation):
         self.h.config["model"]["name"] = "HyraxLoopback"
 
         # Default inference batch size is 512, so this should result in 4 batch files
-        self.h.config["data_set.random_dataset"]["size"] = 4096
-        self.h.config["data_set.random_dataset"]["seed"] = 0
-        self.h.config["data_set.random_dataset"]["shape"] = [self.vector_length]
+        self.h.config["data_set"]["HyraxRandomDataset"]["size"] = 4096
+        self.h.config["data_set"]["HyraxRandomDataset"]["seed"] = 0
+        self.h.config["data_set"]["HyraxRandomDataset"]["shape"] = [1024]
 
         # Create a fake weights file and then run inference on the random dataset
         weights_file = self.input_dir / "fakeweights"
@@ -105,9 +105,9 @@ def setup(self, shard_size_limit, vector_db_implementation):
         self.data_sample = self.ds[4001]["image"].numpy()
 
         self.h.config["vector_db"]["name"] = vector_db_implementation
-        self.h.config["vector_db.chromadb"]["shard_size_limit"] = shard_size_limit
+        self.h.config["vector_db"]["chromadb"]["shard_size_limit"] = shard_size_limit
         # Qdrant requires the vector size in order to create its collections
-        self.h.config["vector_db.qdrant"]["vector_size"] = self.vector_length
+        self.h.config["vector_db"]["qdrant"]["vector_size"] = 4096
 
         # Save inference results to vector database and create a db connection
         self.h.save_to_database(output_dir=Path(self.output_dir))

src/hyrax/data_sets/random/hyrax_random_dataset.py

@@ -51,34 +51,34 @@ def __init__(self, config):
           Valid values are "nan", "inf", "-inf", "none", or a float value.
         """
         # The total number of random data samples produced
-        data_size = config["data_set.random_dataset"]["size"]
+        data_size = config["data_set"]["HyraxRandomDataset"]["size"]
         if not isinstance(data_size, int):
             raise ValueError(
-                f"Expected integer value for `config['data_set.random_dataset']['size']`, but got {data_size}"
+                f"Expected integer for `config['data_set']['random_dataset']['size']`, but got {data_size}"
             )
 
         # The shape of each random data sample as a tuple.
         # i.e. (3, 29, 29) = 3 layers of 2d data, each layer is 29x29 elements.
-        data_shape = tuple(config["data_set.random_dataset"]["shape"])
+        data_shape = tuple(config["data_set"]["HyraxRandomDataset"]["shape"])
         if not len(data_shape):
             raise ValueError(
-                "Expected `config['data_set.random_dataset']['data_shape']` to have at least 1 value."
+                "Expected `config['data_set']['random_dataset']['data_shape']` to have at least 1 value."
             )
 
         for e in data_shape:
             if e < 1:
                 raise ValueError(
-                    f"Expected all values in `config['data_set.random_dataset']['data_shape']`\
+                    f"Expected all values in `config['data_set']['random_dataset']['data_shape']`\
                         to be > 0, but got {data_shape}."
                 )
             if not isinstance(e, int):
                 raise ValueError(
-                    f"Expected all values in `config['data_set.random_dataset']['data_shape']`\
+                    f"Expected all values in `config['data_set']['random_dataset']['data_shape']`\
                         to be integers, but got {data_shape}."
                 )
 
         # Random seed to use for reproducibility
-        seed = config["data_set.random_dataset"]["seed"]
+        seed = config["data_set"]["HyraxRandomDataset"]["seed"]
         rng = np.random.default_rng(seed)
 
         # Note: We raise exceptions if data_size is not an int, so we can assume
@@ -90,23 +90,23 @@ def __init__(self, config):
         self.id_list = list(range(id_start, id_start + data_size))
 
         # Randomly insert flawed values (np.nan, np.inf, -np.inf, None, other float)
-        num_invalid_values = config["data_set.random_dataset"]["number_invalid_values"]
+        num_invalid_values = config["data_set"]["HyraxRandomDataset"]["number_invalid_values"]
         if num_invalid_values:
             # Determine what value to use for invalid values
-            invalid_value_type = config["data_set.random_dataset"]["invalid_value_type"]
+            invalid_value_type = config["data_set"]["HyraxRandomDataset"]["invalid_value_type"]
             if isinstance(invalid_value_type, str):
                 try:
                     invalid_value = INVALID_VALUES[invalid_value_type.lower()]
                 except KeyError as err:
                     raise ValueError(
                         f"Invalid value type '{invalid_value_type}' provided. "
-                        f"Expected `config['data_set.random_dataset']['invalid_value_type']` "
-                        f"to beone of {list(INVALID_VALUES.keys())}"
+                        f"Expected `config['data_set']['random_dataset']['invalid_value_type']` "
+                        f"to be one of {list(INVALID_VALUES.keys())}"
                     ) from err
             else:
                 if not isinstance(invalid_value_type, float):
                     raise ValueError(
-                        f"Expected `config['data_set.random_dataset']['invalid_value_type']` to be "
+                        f"Expected `config['data_set']['random_dataset']['invalid_value_type']` to be "
                         f"a string or a float, but got {type(invalid_value_type)}."
                     )
                 invalid_value = invalid_value_type
@@ -115,7 +115,7 @@ def __init__(self, config):
             flattened[random_inds] = invalid_value
 
         # If a list of possible labels is provided, create the random label list.
-        self.provided_labels = config["data_set.random_dataset"]["provided_labels"]
+        self.provided_labels = config["data_set"]["HyraxRandomDataset"]["provided_labels"]
         if self.provided_labels:
             self.labels = rng.choice(self.provided_labels, size=data_size)
 

src/hyrax/hyrax_default_config.toml

@@ -95,7 +95,7 @@ latent_dim = 64
 final_layer = "tanh"
 
 
-[model.hyrax_cnn]
+[model.HyraxCNN]
 # The number of classes to predict as the output of the model. i.e. 2 would be a
 # binary classifer, 10 would predict the 10 classes in the CiFAR dataset.
 output_classes = 10
@@ -237,7 +237,7 @@ semi_height_deg = 0.00472
 
 
 
-["data_set.random_dataset"]
+[data_set.HyraxRandomDataset]
 # Total number of samples produced by the random dataset
 size = 100
 
@@ -290,7 +290,7 @@ vector_db_dir = false
 infer_results_dir = false
 
 
-["vector_db.chromadb"]
+[vector_db.chromadb]
 # The approximate maximum size of a shard before creating a new one. A smaller
 # value will decrease insert times while increasing search times.
 shard_size_limit = 65536
@@ -300,7 +300,7 @@ shard_size_limit = 65536
 vector_size_warning = 10000
 
 
-["vector_db.qdrant"]
+[vector_db.qdrant]
 # The number of elements in the vectors that will be stored in the vector database.
 # This must be the same as the size of the vectors produced by the model.
 vector_size = 64
@@ -325,7 +325,7 @@ parallel = false
 name = "umap.UMAP"
 
 
-["umap.UMAP"]
+[umap.UMAP]
 # Specify any parameter accepted by https://umap-learn.readthedocs.io/en/latest/api.html#umap
 # Dimension of the embedded space
 n_components = 2

src/hyrax/models/hyrax_cnn.py

@@ -48,7 +48,7 @@ def __init__(self, config, shape=(3, 32, 32)):
         self.conv2 = nn.Conv2d(hidden_channels_1, hidden_channels_2, 5)
         self.fc1 = nn.Linear(hidden_channels_2 * pool2_end_h * pool2_end_w, 120)
         self.fc2 = nn.Linear(120, 84)
-        self.fc3 = nn.Linear(84, self.config["model"]["hyrax_cnn"]["output_classes"])
+        self.fc3 = nn.Linear(84, self.config["model"]["HyraxCNN"]["output_classes"])
 
     def conv2d_output_size(self, input_size, kernel_size, padding=0, stride=1, dilation=1) -> int:
         # From https://pytorch.org/docs/stable/generated/torch.nn.Conv2d.html

src/hyrax/vector_dbs/chromadb_impl.py

@@ -77,10 +77,10 @@ def __init__(self, config, context):
         self.shard_size = 0  # The number of vectors in the current shard
 
         # The approximate maximum size of a shard before a new one is created
-        self.shard_size_limit = self.config["vector_db.chromadb"]["shard_size_limit"]
+        self.shard_size_limit = self.config["vector_db"]["chromadb"]["shard_size_limit"]
 
         # If set, inserting a vector with number of elements >= this logs a warning.
-        self.vector_size_limit = self.config["vector_db.chromadb"]["vector_size_warning"]
+        self.vector_size_limit = self.config["vector_db"]["chromadb"]["vector_size_warning"]
 
         # Min number of shards before using multiprocess to parallelize the search
         self.min_shards_for_parallelization = MIN_SHARDS_FOR_PARALLELIZATION
@@ -144,7 +144,7 @@ def insert(self, ids: list[Union[str, int]], vectors: list[np.ndarray]):
             logger.warning(
                 f"Attempting to insert vectors with length: {len(vectors[0])}.\
                            Chroma DB often has poor performance when working with vectors\
-                           larger than {self.config['vector_db.chromadb']['vector_size_warning']}"
+                           larger than {self.config['vector_db']['chromadb']['vector_size_warning']}"
             )
 
         # increment counter, if exceeds shard limit, create a new collection

src/hyrax/vector_dbs/qdrantdb_impl.py

@@ -58,7 +58,7 @@ def create(self):
                     #! This stinks - we should just check the size of the data
                     #! when we call `save_to_database` and then set this automatically
                     #! as a parameter in self.context["blah"] or something.
-                    size=self.config["vector_db.qdrant"]["vector_size"],
+                    size=self.config["vector_db"]["qdrant"]["vector_size"],
                     distance=models.Distance.EUCLID,
                     on_disk=True,
                 ),
@@ -85,7 +85,7 @@ def insert(self, ids: list[Union[str, int]], vectors: list[np.ndarray]):
         if self.client is None:
             self.connect()
 
-        expected_size = self.config["vector_db.qdrant"]["vector_size"]
+        expected_size = self.config["vector_db"]["qdrant"]["vector_size"]
         for idx, vector in enumerate(vectors):
             if len(vector) != expected_size:
                 raise ValueError(

src/hyrax/verbs/umap.py

@@ -78,7 +78,7 @@ def _run(self, input_dir: Optional[Union[Path, str]] = None):
         from hyrax.config_utils import create_results_dir
         from hyrax.data_sets.inference_dataset import InferenceDataSet, InferenceDataSetWriter
 
-        self.reducer = umap.UMAP(**self.config["umap.UMAP"])
+        self.reducer = umap.UMAP(**self.config["umap"]["UMAP"])
 
         # Load all the latent space data.
         inference_results = InferenceDataSet(self.config, results_dir=input_dir)

tests/hyrax/conftest.py

@@ -105,9 +105,9 @@ def loopback_hyrax(tmp_path_factory, request):
 
     h.config["general"]["dev_mode"] = True
     h.config["data_set"]["name"] = request.param
-    h.config["data_set.random_dataset"]["size"] = 20
-    h.config["data_set.random_dataset"]["seed"] = 0
-    h.config["data_set.random_dataset"]["shape"] = [2, 3]
+    h.config["data_set"]["HyraxRandomDataset"]["size"] = 20
+    h.config["data_set"]["HyraxRandomDataset"]["seed"] = 0
+    h.config["data_set"]["HyraxRandomDataset"]["shape"] = [2, 3]
 
     h.config["data_set"]["validate_size"] = 0.2
     h.config["data_set"]["test_size"] = 0.2

tests/hyrax/test_chromadb_impl.py

@@ -102,7 +102,7 @@ def test_insert_does_not_raise_warning(caplog, tmp_path, random_vector_generator
     when the config vector_size_warning is set to `False`."""
 
     h = Hyrax()
-    h.config["vector_db.chromadb"]["vector_size_warning"] = False
+    h.config["vector_db"]["chromadb"]["vector_size_warning"] = False
     chromadb_instance = ChromaDB(h.config, {"results_dir": tmp_path})
     chromadb_instance.connect()
     chromadb_instance.create()

tests/hyrax/test_nan.py

@@ -34,11 +34,11 @@ def loopback_hyrax_nan(tmp_path_factory, request):
 
     h.config["general"]["dev_mode"] = True
     h.config["data_set"]["name"] = request.param
-    h.config["data_set.random_dataset"]["size"] = 20
-    h.config["data_set.random_dataset"]["seed"] = 0
-    h.config["data_set.random_dataset"]["shape"] = [2, 3]
-    h.config["data_set.random_dataset"]["number_invalid_values"] = 40
-    h.config["data_set.random_dataset"]["invalid_value_type"] = "nan"
+    h.config["data_set"]["HyraxRandomDataset"]["size"] = 20
+    h.config["data_set"]["HyraxRandomDataset"]["seed"] = 0
+    h.config["data_set"]["HyraxRandomDataset"]["shape"] = [2, 3]
+    h.config["data_set"]["HyraxRandomDataset"]["number_invalid_values"] = 40
+    h.config["data_set"]["HyraxRandomDataset"]["invalid_value_type"] = "nan"
 
     h.config["data_set"]["validate_size"] = 0.2
     h.config["data_set"]["test_size"] = 0.2