Skip to content

Vb/fact checking plt 1709 #1883

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Nov 20, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions docs/labelbox/fact-checking-tool.rst
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
Fact Checking Tool
===============================================================================================

.. automodule:: labelbox.schema.tool_building.fact_checking_tool
:members:
:show-inheritance:
1 change: 1 addition & 0 deletions docs/labelbox/index.rst
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ Labelbox Python SDK Documentation
enums
exceptions
export-task
fact-checking-tool
foundry-client
foundry-model
identifiable
Expand Down
4 changes: 2 additions & 2 deletions libs/labelbox/src/labelbox/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -55,8 +55,8 @@
ResponseOption,
Tool,
)
from labelbox.schema.ontology import PromptResponseClassification
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

removed dupes

from labelbox.schema.ontology import ResponseOption
from labelbox.schema.tool_building.fact_checking_tool import FactCheckingTool
from labelbox.schema.tool_building.step_reasoning_tool import StepReasoningTool
from labelbox.schema.role import Role, ProjectRole
from labelbox.schema.invite import Invite, InviteLimit
from labelbox.schema.data_row_metadata import (
Expand Down
20 changes: 16 additions & 4 deletions libs/labelbox/src/labelbox/schema/ontology.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,12 @@

from labelbox.orm.db_object import DbObject
from labelbox.orm.model import Field, Relationship
from labelbox.schema.tool_building.fact_checking_tool import FactCheckingTool
from labelbox.schema.tool_building.step_reasoning_tool import StepReasoningTool
from labelbox.schema.tool_building.tool_type import ToolType
from labelbox.schema.tool_building.tool_type_mapping import (
map_tool_type_to_tool_cls,
)

FeatureSchemaId: Type[str] = Annotated[
str, StringConstraints(min_length=25, max_length=25)
Expand Down Expand Up @@ -490,14 +494,20 @@ def add_classification(self, classification: Classification) -> None:
self.classifications.append(classification)


"""
The following 2 functions help to bridge the gap between the step reasoning all other tool ontologies.
"""


def tool_cls_from_type(tool_type: str):
if tool_type.lower() == ToolType.STEP_REASONING.value:
return StepReasoningTool
tool_cls = map_tool_type_to_tool_cls(tool_type)
if tool_cls is not None:
return tool_cls
return Tool


def tool_type_cls_from_type(tool_type: str):
if tool_type.lower() == ToolType.STEP_REASONING.value:
if ToolType.valid(tool_type):
return ToolType
return Tool.Type

Expand Down Expand Up @@ -596,7 +606,9 @@ class OntologyBuilder:

"""

tools: List[Union[Tool, StepReasoningTool]] = field(default_factory=list)
tools: List[Union[Tool, StepReasoningTool, FactCheckingTool]] = field(
default_factory=list
)
classifications: List[
Union[Classification, PromptResponseClassification]
] = field(default_factory=list)
Expand Down
2 changes: 2 additions & 0 deletions libs/labelbox/src/labelbox/schema/tool_building/__init__.py
Original file line number Diff line number Diff line change
@@ -1,2 +1,4 @@
import labelbox.schema.tool_building.tool_type
import labelbox.schema.tool_building.step_reasoning_tool
import labelbox.schema.tool_building.fact_checking_tool
import labelbox.schema.tool_building.tool_type_mapping
Original file line number Diff line number Diff line change
@@ -0,0 +1,109 @@
import warnings
from abc import ABC
from dataclasses import dataclass, field
from typing import Any, Dict, List, Optional, Set

from labelbox.schema.tool_building.tool_type import ToolType


@dataclass
class _Variant:
id: int
name: str
actions: List[str] = field(default_factory=list)
_available_actions: Set[str] = field(default_factory=set)

def set_actions(self, actions: List[str]) -> None:
self.actions = []
for action in actions:
if action in self._available_actions:
self.actions.append(action)
else:
warnings.warn(
f"Variant ID {self.id} {action} is an invalid action, skipping"
)

def reset_actions(self) -> None:
self.actions = []

def asdict(self) -> Dict[str, Any]:
return {
"id": self.id,
"name": self.name,
"actions": self.actions,
}

def _post_init(self):
# Call set_actions to remove any invalid actions
self.set_actions(self.actions)


@dataclass
class _Definition:
variants: List[_Variant]
version: int = field(default=1)
title: Optional[str] = None
value: Optional[str] = None

def __post_init__(self):
if self.version != 1:
raise ValueError("Invalid version")

def asdict(self) -> Dict[str, Any]:
result = {
"variants": [variant.asdict() for variant in self.variants],
"version": self.version,
}
if self.title is not None:
result["title"] = self.title
if self.value is not None:
result["value"] = self.value
return result

@classmethod
def from_dict(cls, dictionary: Dict[str, Any]) -> "_Definition":
variants = [_Variant(**variant) for variant in dictionary["variants"]]
title = dictionary.get("title", None)
value = dictionary.get("value", None)
return cls(variants=variants, title=title, value=value)


@dataclass
class _BaseStepReasoningTool(ABC):
name: str
definition: _Definition
type: Optional[ToolType] = None
schema_id: Optional[str] = None
feature_schema_id: Optional[str] = None
color: Optional[str] = None
required: bool = False

def __post_init__(self):
warnings.warn(
"This feature is experimental and subject to change.",
)

if not self.name.strip():
raise ValueError("Name is required")

def asdict(self) -> Dict[str, Any]:
return {
"tool": self.type.value if self.type else None,
"name": self.name,
"required": self.required,
"schemaNodeId": self.schema_id,
"featureSchemaId": self.feature_schema_id,
"definition": self.definition.asdict(),
"color": self.color,
}

@classmethod
def from_dict(cls, dictionary: Dict[str, Any]) -> "_BaseStepReasoningTool":
return cls(
name=dictionary["name"],
schema_id=dictionary.get("schemaNodeId", None),
feature_schema_id=dictionary.get("featureSchemaId", None),
required=dictionary.get("required", False),
definition=_Definition.from_dict(dictionary["definition"]),
color=dictionary.get("color", None),
)
Original file line number Diff line number Diff line change
@@ -0,0 +1,84 @@
from dataclasses import dataclass, field
from enum import Enum

from labelbox.schema.tool_building.base_step_reasoning_tool import (
_BaseStepReasoningTool,
_Definition,
_Variant,
)
from labelbox.schema.tool_building.tool_type import ToolType


class FactCheckingActions(Enum):
WRITE_JUSTIFICATION = "justification"


def build_fact_checking_definition():
accurate_step = _Variant(
id=0,
name="Accurate",
_available_actions={action.value for action in FactCheckingActions},
actions=[action.value for action in FactCheckingActions],
)
inaccurate_step = _Variant(
id=1,
name="Inaccurate",
_available_actions={action.value for action in FactCheckingActions},
actions=[action.value for action in FactCheckingActions],
)
disputed_step = _Variant(
id=2,
name="Disputed",
_available_actions={action.value for action in FactCheckingActions},
actions=[action.value for action in FactCheckingActions],
)
unsupported_step = _Variant(
id=3,
name="Unsupported",
_available_actions=set(),
actions=[],
)
cant_confidently_assess_step = _Variant(
id=4,
name="Can't confidently assess",
_available_actions=set(),
actions=[],
)
no_factual_information_step = _Variant(
id=5,
name="No factual information",
_available_actions=set(),
actions=[],
)
variants = [
accurate_step,
inaccurate_step,
disputed_step,
unsupported_step,
cant_confidently_assess_step,
no_factual_information_step,
]
return _Definition(variants=variants)


@dataclass
class FactCheckingTool(_BaseStepReasoningTool):
"""
Use this class in OntologyBuilder to create a tool for fact checking
"""

type: ToolType = field(default=ToolType.FACT_CHECKING, init=False)
definition: _Definition = field(
default_factory=build_fact_checking_definition
)

def __post_init__(self):
super().__post_init__()
# Set available actions for variants 0, 1, 2 'out of band' since they are not passed in the definition
self._set_variant_available_actions()

def _set_variant_available_actions(self):
for variant in self.definition.variants:
if variant.id in [0, 1, 2]:
for action in FactCheckingActions:
variant._available_actions.add(action.value)
Loading
Loading