Read hdf returns unexpected values for categorical

doc/source/whatsnew/v1.3.0.rst

@@ -330,6 +330,7 @@ I/O
 - Bug in :func:`read_csv` not switching ``true_values`` and ``false_values`` for nullable ``boolean`` dtype (:issue:`34655`)
 - Bug in :func:`read_json` when ``orient="split"`` does not maintain numeric string index (:issue:`28556`)
 - :meth:`read_sql` returned an empty generator if ``chunksize`` was no-zero and the query returned no results. Now returns a generator with a single empty dataframe (:issue:`34411`)
+- Bug in :func:`read_hdf` returning unexpected records when filtering on categorical string columns using ``where`` parameter (:issue:`39189`)
 
 Period
 ^^^^^^

pandas/core/computation/pytables.py

@@ -19,6 +19,7 @@
 from pandas.core.computation.ops import UndefinedVariableError, is_term
 from pandas.core.construction import extract_array
 from pandas.core.indexes.base import Index
+from pandas.core.series import Series
 
 from pandas.io.formats.printing import pprint_thing, pprint_thing_encoded
 
@@ -209,14 +210,8 @@ def stringify(value):
                 v = Timedelta(v, unit="s").value
             return TermValue(int(v), v, kind)
         elif meta == "category":
-            metadata = extract_array(self.metadata, extract_numpy=True)
-            result = metadata.searchsorted(v, side="left")
-
-            # result returns 0 if v is first element or if v is not in metadata
-            # check that metadata contains v
-            if not result and v not in metadata:
-                result = -1
-            return TermValue(result, result, "integer")
+            term_value = self._convert_category_value(self.metadata, v)
+            return term_value
         elif kind == "integer":
             v = int(float(v))
             return TermValue(v, v, kind)
@@ -245,6 +240,15 @@ def stringify(value):
         else:
             raise TypeError(f"Cannot compare {v} of type {type(v)} to {kind} column")
 
+    @staticmethod
+    def _convert_category_value(metadata: Series, value: Any) -> TermValue:
+        metadata = extract_array(metadata, extract_numpy=True)
+        if value not in metadata:
+            result = -1
+        else:
+            result = metadata.searchsorted(value, side="left")
+        return TermValue(result, result, "integer")
+
     def convert_values(self):
         pass
 

pandas/tests/computation/test_pytables.py

@@ -0,0 +1,23 @@
+from typing import Any
+
+import numpy as np
+import pytest
+
+from pandas import Series
+from pandas.core.computation.pytables import BinOp, TermValue
+
+
+@pytest.mark.parametrize(
+    "value, expected_results",
+    [("q", TermValue(-1, -1, "integer")), ("a", TermValue(0, 0, "integer"))],
+)
+def test__convert_value(value: Any, expected_results: TermValue):
+    metadata = Series(np.array(["a", "b", "s"]))
+
+    result = BinOp._convert_category_value(metadata, value)
+
+    assert (
+        result.kind == expected_results.kind
+        and result.value == expected_results.value
+        and result.converted == expected_results.converted
+    )