Add FactCheckingTool

Refactor StepReasoning to also reuse Variants

Author: vbrodsky

docs/labelbox/fact-checking-tool.rst

@@ -0,0 +1,6 @@
+Fact Checking Tool
+===============================================================================================
+
+.. automodule:: labelbox.schema.tool_building.fact_checking_tool
+   :members:
+   :show-inheritance:

docs/labelbox/index.rst

@@ -19,6 +19,7 @@ Labelbox Python SDK Documentation
     enums
     exceptions
     export-task
+    fact-checking-tool
     foundry-client
     foundry-model
     identifiable

libs/labelbox/src/labelbox/__init__.py

@@ -55,8 +55,8 @@
     ResponseOption,
     Tool,
 )
-from labelbox.schema.ontology import PromptResponseClassification
-from labelbox.schema.ontology import ResponseOption
+from labelbox.schema.tool_building.fact_checking_tool import FactCheckingTool
+from labelbox.schema.tool_building.step_reasoning_tool import StepReasoningTool
 from labelbox.schema.role import Role, ProjectRole
 from labelbox.schema.invite import Invite, InviteLimit
 from labelbox.schema.data_row_metadata import (

libs/labelbox/src/labelbox/schema/ontology.py

@@ -12,8 +12,12 @@
 
 from labelbox.orm.db_object import DbObject
 from labelbox.orm.model import Field, Relationship
+from labelbox.schema.tool_building.fact_checking_tool import FactCheckingTool
 from labelbox.schema.tool_building.step_reasoning_tool import StepReasoningTool
 from labelbox.schema.tool_building.tool_type import ToolType
+from labelbox.schema.tool_building.tool_type_mapping import (
+    map_tool_type_to_tool_cls,
+)
 
 FeatureSchemaId: Type[str] = Annotated[
     str, StringConstraints(min_length=25, max_length=25)
@@ -490,14 +494,20 @@ def add_classification(self, classification: Classification) -> None:
         self.classifications.append(classification)
 
 
+"""
+The following 2 functions help to bridge the gap between the step reasoning all other tool ontologies.
+"""
+
+
 def tool_cls_from_type(tool_type: str):
-    if tool_type.lower() == ToolType.STEP_REASONING.value:
-        return StepReasoningTool
+    tool_cls = map_tool_type_to_tool_cls(tool_type)
+    if tool_cls is not None:
+        return tool_cls
     return Tool
 
 
 def tool_type_cls_from_type(tool_type: str):
-    if tool_type.lower() == ToolType.STEP_REASONING.value:
+    if ToolType.valid(tool_type):
         return ToolType
     return Tool.Type
 
@@ -596,7 +606,9 @@ class OntologyBuilder:
 
     """
 
-    tools: List[Union[Tool, StepReasoningTool]] = field(default_factory=list)
+    tools: List[Union[Tool, StepReasoningTool, FactCheckingTool]] = field(
+        default_factory=list
+    )
     classifications: List[
         Union[Classification, PromptResponseClassification]
     ] = field(default_factory=list)

libs/labelbox/src/labelbox/schema/tool_building/__init__.py

@@ -1,2 +1,4 @@
 import labelbox.schema.tool_building.tool_type
 import labelbox.schema.tool_building.step_reasoning_tool
+import labelbox.schema.tool_building.fact_checking_tool
+import labelbox.schema.tool_building.tool_type_mapping

libs/labelbox/src/labelbox/schema/tool_building/base_step_reasoning_tool.py

@@ -0,0 +1,109 @@
+import warnings
+from abc import ABC
+from dataclasses import dataclass, field
+from typing import Any, Dict, List, Optional, Set
+
+from labelbox.schema.tool_building.tool_type import ToolType
+
+
+@dataclass
+class _Variant:
+    id: int
+    name: str
+    actions: List[str] = field(default_factory=list)
+    _available_actions: Set[str] = field(default_factory=set)
+
+    def set_actions(self, actions: List[str]) -> None:
+        self.actions = []
+        for action in actions:
+            if action in self._available_actions:
+                self.actions.append(action)
+            else:
+                warnings.warn(
+                    f"Variant ID {self.id} {action} is an invalid action, skipping"
+                )
+
+    def reset_actions(self) -> None:
+        self.actions = []
+
+    def asdict(self) -> Dict[str, Any]:
+        return {
+            "id": self.id,
+            "name": self.name,
+            "actions": self.actions,
+        }
+
+    def _post_init(self):
+        # Call set_actions to remove any invalid actions
+        self.set_actions(self.actions)
+
+
+@dataclass
+class _Definition:
+    variants: List[_Variant]
+    version: int = field(default=1)
+    title: Optional[str] = None
+    value: Optional[str] = None
+
+    def __post_init__(self):
+        if self.version != 1:
+            raise ValueError("Invalid version")
+
+    def asdict(self) -> Dict[str, Any]:
+        result = {
+            "variants": [variant.asdict() for variant in self.variants],
+            "version": self.version,
+        }
+        if self.title is not None:
+            result["title"] = self.title
+        if self.value is not None:
+            result["value"] = self.value
+        return result
+
+    @classmethod
+    def from_dict(cls, dictionary: Dict[str, Any]) -> "_Definition":
+        variants = [_Variant(**variant) for variant in dictionary["variants"]]
+        title = dictionary.get("title", None)
+        value = dictionary.get("value", None)
+        return cls(variants=variants, title=title, value=value)
+
+
+@dataclass
+class _BaseStepReasoningTool(ABC):
+    name: str
+    definition: _Definition
+    type: Optional[ToolType] = None
+    schema_id: Optional[str] = None
+    feature_schema_id: Optional[str] = None
+    color: Optional[str] = None
+    required: bool = False
+
+    def __post_init__(self):
+        warnings.warn(
+            "This feature is experimental and subject to change.",
+        )
+
+        if not self.name.strip():
+            raise ValueError("Name is required")
+
+    def asdict(self) -> Dict[str, Any]:
+        return {
+            "tool": self.type.value if self.type else None,
+            "name": self.name,
+            "required": self.required,
+            "schemaNodeId": self.schema_id,
+            "featureSchemaId": self.feature_schema_id,
+            "definition": self.definition.asdict(),
+            "color": self.color,
+        }
+
+    @classmethod
+    def from_dict(cls, dictionary: Dict[str, Any]) -> "_BaseStepReasoningTool":
+        return cls(
+            name=dictionary["name"],
+            schema_id=dictionary.get("schemaNodeId", None),
+            feature_schema_id=dictionary.get("featureSchemaId", None),
+            required=dictionary.get("required", False),
+            definition=_Definition.from_dict(dictionary["definition"]),
+            color=dictionary.get("color", None),
+        )

libs/labelbox/src/labelbox/schema/tool_building/fact_checking_tool.py

@@ -0,0 +1,84 @@
+from dataclasses import dataclass, field
+from enum import Enum
+
+from labelbox.schema.tool_building.base_step_reasoning_tool import (
+    _BaseStepReasoningTool,
+    _Definition,
+    _Variant,
+)
+from labelbox.schema.tool_building.tool_type import ToolType
+
+
+class FactCheckingActions(Enum):
+    WRITE_JUSTIFICATION = "justification"
+
+
+def build_fact_checking_definition():
+    accurate_step = _Variant(
+        id=0,
+        name="Accurate",
+        _available_actions={action.value for action in FactCheckingActions},
+        actions=[action.value for action in FactCheckingActions],
+    )
+    inaccurate_step = _Variant(
+        id=1,
+        name="Inaccurate",
+        _available_actions={action.value for action in FactCheckingActions},
+        actions=[action.value for action in FactCheckingActions],
+    )
+    disputed_step = _Variant(
+        id=2,
+        name="Disputed",
+        _available_actions={action.value for action in FactCheckingActions},
+        actions=[action.value for action in FactCheckingActions],
+    )
+    unsupported_step = _Variant(
+        id=3,
+        name="Unsupported",
+        _available_actions=set(),
+        actions=[],
+    )
+    cant_confidently_assess_step = _Variant(
+        id=4,
+        name="Can't confidently assess",
+        _available_actions=set(),
+        actions=[],
+    )
+    no_factual_information_step = _Variant(
+        id=5,
+        name="No factual information",
+        _available_actions=set(),
+        actions=[],
+    )
+    variants = [
+        accurate_step,
+        inaccurate_step,
+        disputed_step,
+        unsupported_step,
+        cant_confidently_assess_step,
+        no_factual_information_step,
+    ]
+    return _Definition(variants=variants)
+
+
+@dataclass
+class FactCheckingTool(_BaseStepReasoningTool):
+    """
+    Use this class in OntologyBuilder to create a tool for fact checking
+    """
+
+    type: ToolType = field(default=ToolType.FACT_CHECKING, init=False)
+    definition: _Definition = field(
+        default_factory=build_fact_checking_definition
+    )
+
+    def __post_init__(self):
+        super().__post_init__()
+        # Set available actions for variants 0, 1, 2 'out of band' since they are not passed in the definition
+        self._set_variant_available_actions()
+
+    def _set_variant_available_actions(self):
+        for variant in self.definition.variants:
+            if variant.id in [0, 1, 2]:
+                for action in FactCheckingActions:
+                    variant._available_actions.add(action.value)