diff --git a/docs/labelbox/index.rst b/docs/labelbox/index.rst index f28de02fe..41207c78f 100644 --- a/docs/labelbox/index.rst +++ b/docs/labelbox/index.rst @@ -40,6 +40,7 @@ Labelbox Python SDK Documentation pagination project project-model-config + prompt-issue-tool quality-mode request-client resource-tag @@ -47,7 +48,7 @@ Labelbox Python SDK Documentation search-filters send-to-annotate-params slice - step_reasoning_tool + step-reasoning-tool task task-queue user diff --git a/docs/labelbox/prompt-issue-tool.rst b/docs/labelbox/prompt-issue-tool.rst new file mode 100644 index 000000000..7f1842e4b --- /dev/null +++ b/docs/labelbox/prompt-issue-tool.rst @@ -0,0 +1,6 @@ +Step Reasoning Tool +=============================================================================================== + +.. automodule:: labelbox.schema.tool_building.prompt_issue_tool + :members: + :show-inheritance: \ No newline at end of file diff --git a/docs/labelbox/step_reasoning_tool.rst b/docs/labelbox/step-reasoning-tool.rst similarity index 100% rename from docs/labelbox/step_reasoning_tool.rst rename to docs/labelbox/step-reasoning-tool.rst diff --git a/libs/labelbox/src/labelbox/__init__.py b/libs/labelbox/src/labelbox/__init__.py index 74fc047ed..9d43ab241 100644 --- a/libs/labelbox/src/labelbox/__init__.py +++ b/libs/labelbox/src/labelbox/__init__.py @@ -46,17 +46,14 @@ from labelbox.schema.model_config import ModelConfig from labelbox.schema.model_run import DataSplit, ModelRun from labelbox.schema.ontology import ( - Classification, FeatureSchema, Ontology, OntologyBuilder, - Option, - PromptResponseClassification, - ResponseOption, Tool, ) from labelbox.schema.tool_building.fact_checking_tool import FactCheckingTool from labelbox.schema.tool_building.step_reasoning_tool import StepReasoningTool +from labelbox.schema.tool_building.prompt_issue_tool import PromptIssueTool from labelbox.schema.role import Role, ProjectRole from labelbox.schema.invite import Invite, InviteLimit from labelbox.schema.data_row_metadata import ( @@ -94,3 +91,9 @@ from labelbox.schema.task_queue import TaskQueue from labelbox.schema.user import User from labelbox.schema.webhook import Webhook +from labelbox.schema.tool_building.classification import ( + Classification, + Option, + ResponseOption, + PromptResponseClassification, +) diff --git a/libs/labelbox/src/labelbox/schema/data_row_metadata.py b/libs/labelbox/src/labelbox/schema/data_row_metadata.py index 58111828f..2140fa5c3 100644 --- a/libs/labelbox/src/labelbox/schema/data_row_metadata.py +++ b/libs/labelbox/src/labelbox/schema/data_row_metadata.py @@ -29,7 +29,7 @@ from labelbox.schema.identifiable import GlobalKey, UniqueId from labelbox.schema.identifiables import DataRowIdentifiers, UniqueIds -from labelbox.schema.ontology import SchemaId +from labelbox.schema.tool_building.types import SchemaId from labelbox.utils import ( _CamelCaseMixin, format_iso_datetime, diff --git a/libs/labelbox/src/labelbox/schema/labeling_service_dashboard.py b/libs/labelbox/src/labelbox/schema/labeling_service_dashboard.py index 9179a4665..9899212e5 100644 --- a/libs/labelbox/src/labelbox/schema/labeling_service_dashboard.py +++ b/libs/labelbox/src/labelbox/schema/labeling_service_dashboard.py @@ -71,13 +71,17 @@ class LabelingServiceDashboard(_CamelCaseMixin): created_at: Optional[datetime] = Field(frozen=True, default=None) updated_at: Optional[datetime] = Field(frozen=True, default=None) created_by_id: Optional[str] = Field(frozen=True, default=None) - status: LabelingServiceStatus = Field(frozen=True, default=None) + status: Optional[LabelingServiceStatus] = Field(frozen=True, default=None) data_rows_count: int = Field(frozen=True) tasks_completed_count: int = Field(frozen=True) tasks_remaining_count: Optional[int] = Field(frozen=True, default=None) media_type: Optional[MediaType] = Field(frozen=True, default=None) - editor_task_type: EditorTaskType = Field(frozen=True, default=None) - tags: List[LabelingServiceDashboardTags] = Field(frozen=True, default=None) + editor_task_type: Optional[EditorTaskType] = Field( + frozen=True, default=None + ) + tags: Optional[List[LabelingServiceDashboardTags]] = Field( + frozen=True, default=None + ) client: Any # type Any to avoid circular import from client diff --git a/libs/labelbox/src/labelbox/schema/ontology.py b/libs/labelbox/src/labelbox/schema/ontology.py index 607bf8fec..0032aaad1 100644 --- a/libs/labelbox/src/labelbox/schema/ontology.py +++ b/libs/labelbox/src/labelbox/schema/ontology.py @@ -2,30 +2,30 @@ import colorsys import json -import warnings from dataclasses import dataclass, field from enum import Enum -from typing import Annotated, Any, Dict, List, Optional, Type, Union +from typing import Any, Dict, List, Optional, Union from lbox.exceptions import InconsistentOntologyException -from pydantic import StringConstraints from labelbox.orm.db_object import DbObject from labelbox.orm.model import Field, Relationship -from labelbox.schema.tool_building.fact_checking_tool import FactCheckingTool -from labelbox.schema.tool_building.step_reasoning_tool import StepReasoningTool +from labelbox.schema.tool_building.classification import ( + Classification, + PromptResponseClassification, +) +from labelbox.schema.tool_building.fact_checking_tool import ( + FactCheckingTool, +) +from labelbox.schema.tool_building.prompt_issue_tool import PromptIssueTool +from labelbox.schema.tool_building.step_reasoning_tool import ( + StepReasoningTool, +) from labelbox.schema.tool_building.tool_type import ToolType from labelbox.schema.tool_building.tool_type_mapping import ( map_tool_type_to_tool_cls, ) -FeatureSchemaId: Type[str] = Annotated[ - str, StringConstraints(min_length=25, max_length=25) -] -SchemaId: Type[str] = Annotated[ - str, StringConstraints(min_length=25, max_length=25) -] - class DeleteFeatureFromOntologyResult: archived: bool @@ -44,367 +44,6 @@ class FeatureSchema(DbObject): normalized = Field.Json("normalized") -@dataclass -class Option: - """ - An option is a possible answer within a Classification object in - a Project's ontology. - - To instantiate, only the "value" parameter needs to be passed in. - - Example(s): - option = Option(value = "Option Example") - - Attributes: - value: (str) - schema_id: (str) - feature_schema_id: (str) - options: (list) - """ - - value: Union[str, int] - label: Optional[Union[str, int]] = None - schema_id: Optional[str] = None - feature_schema_id: Optional[FeatureSchemaId] = None - options: Union[ - List["Classification"], List["PromptResponseClassification"] - ] = field(default_factory=list) - - def __post_init__(self): - if self.label is None: - self.label = self.value - - @classmethod - def from_dict( - cls, dictionary: Dict[str, Any] - ) -> Dict[Union[str, int], Union[str, int]]: - return cls( - value=dictionary["value"], - label=dictionary["label"], - schema_id=dictionary.get("schemaNodeId", None), - feature_schema_id=dictionary.get("featureSchemaId", None), - options=[ - Classification.from_dict(o) - for o in dictionary.get("options", []) - ], - ) - - def asdict(self) -> Dict[str, Any]: - return { - "schemaNodeId": self.schema_id, - "featureSchemaId": self.feature_schema_id, - "label": self.label, - "value": self.value, - "options": [o.asdict(is_subclass=True) for o in self.options], - } - - def add_option( - self, option: Union["Classification", "PromptResponseClassification"] - ) -> None: - if option.name in (o.name for o in self.options): - raise InconsistentOntologyException( - f"Duplicate nested classification '{option.name}' " - f"for option '{self.label}'" - ) - self.options.append(option) - - -@dataclass -class Classification: - """ - A classification to be added to a Project's ontology. The - classification is dependent on the Classification Type. - - To instantiate, the "class_type" and "name" parameters must - be passed in. - - The "options" parameter holds a list of Option objects. This is not - necessary for some Classification types, such as TEXT. To see which - types require options, look at the "_REQUIRES_OPTIONS" class variable. - - Example(s): - classification = Classification( - class_type = Classification.Type.TEXT, - name = "Classification Example") - - classification_two = Classification( - class_type = Classification.Type.RADIO, - name = "Second Example") - classification_two.add_option(Option( - value = "Option Example")) - - Attributes: - class_type: (Classification.Type) - name: (str) - instructions: (str) - required: (bool) - options: (list) - ui_mode: (str) - schema_id: (str) - feature_schema_id: (str) - scope: (str) - """ - - class Type(Enum): - TEXT = "text" - CHECKLIST = "checklist" - RADIO = "radio" - - class Scope(Enum): - GLOBAL = "global" - INDEX = "index" - - class UIMode(Enum): - HOTKEY = "hotkey" - SEARCHABLE = "searchable" - - _REQUIRES_OPTIONS = {Type.CHECKLIST, Type.RADIO} - - class_type: Type - name: Optional[str] = None - instructions: Optional[str] = None - required: bool = False - options: List[Option] = field(default_factory=list) - schema_id: Optional[str] = None - feature_schema_id: Optional[str] = None - scope: Scope = None - ui_mode: Optional[UIMode] = ( - None # How this classification should be answered (e.g. hotkeys / autocomplete, etc) - ) - - def __post_init__(self): - if self.name is None: - msg = ( - "When creating the Classification feature, please use “name” " - "for the classification schema name, which will be used when " - "creating annotation payload for Model-Assisted Labeling " - "Import and Label Import. “instructions” is no longer " - "supported to specify classification schema name." - ) - if self.instructions is not None: - self.name = self.instructions - warnings.warn(msg) - else: - raise ValueError(msg) - else: - if self.instructions is None: - self.instructions = self.name - - @classmethod - def from_dict(cls, dictionary: Dict[str, Any]) -> Dict[str, Any]: - return cls( - class_type=Classification.Type(dictionary["type"]), - name=dictionary["name"], - instructions=dictionary["instructions"], - required=dictionary.get("required", False), - options=[Option.from_dict(o) for o in dictionary["options"]], - ui_mode=cls.UIMode(dictionary["uiMode"]) - if "uiMode" in dictionary - else None, - schema_id=dictionary.get("schemaNodeId", None), - feature_schema_id=dictionary.get("featureSchemaId", None), - scope=cls.Scope(dictionary.get("scope", cls.Scope.GLOBAL)), - ) - - def asdict(self, is_subclass: bool = False) -> Dict[str, Any]: - if self.class_type in self._REQUIRES_OPTIONS and len(self.options) < 1: - raise InconsistentOntologyException( - f"Classification '{self.name}' requires options." - ) - classification = { - "type": self.class_type.value, - "instructions": self.instructions, - "name": self.name, - "required": self.required, - "options": [o.asdict() for o in self.options], - "schemaNodeId": self.schema_id, - "featureSchemaId": self.feature_schema_id, - } - if ( - self.class_type == self.Type.RADIO - or self.class_type == self.Type.CHECKLIST - ) and self.ui_mode: - # added because this key does nothing for text so no point of including - classification["uiMode"] = self.ui_mode.value - if is_subclass: - return classification - classification["scope"] = ( - self.scope.value - if self.scope is not None - else self.Scope.GLOBAL.value - ) - return classification - - def add_option(self, option: Option) -> None: - if option.value in (o.value for o in self.options): - raise InconsistentOntologyException( - f"Duplicate option '{option.value}' " - f"for classification '{self.name}'." - ) - self.options.append(option) - - -@dataclass -class ResponseOption(Option): - """ - An option is a possible answer within a PromptResponseClassification response object in - a Project's ontology. - - To instantiate, only the "value" parameter needs to be passed in. - - Example(s): - option = ResponseOption(value = "Response Option Example") - - Attributes: - value: (str) - schema_id: (str) - feature_schema_id: (str) - options: (list) - """ - - @classmethod - def from_dict( - cls, dictionary: Dict[str, Any] - ) -> Dict[Union[str, int], Union[str, int]]: - return cls( - value=dictionary["value"], - label=dictionary["label"], - schema_id=dictionary.get("schemaNodeId", None), - feature_schema_id=dictionary.get("featureSchemaId", None), - options=[ - PromptResponseClassification.from_dict(o) - for o in dictionary.get("options", []) - ], - ) - - -@dataclass -class PromptResponseClassification: - """ - - A PromptResponseClassification to be added to a Project's ontology. The - classification is dependent on the PromptResponseClassification Type. - - To instantiate, the "class_type" and "name" parameters must - be passed in. - - The "options" parameter holds a list of Response Option objects. This is not - necessary for some Classification types, such as RESPONSE_TEXT or PROMPT. To see which - types require options, look at the "_REQUIRES_OPTIONS" class variable. - - Example(s): - >>> classification = PromptResponseClassification( - >>> class_type = PromptResponseClassification.Type.Prompt, - >>> character_min = 1, - >>> character_max = 1 - >>> name = "Prompt Classification Example") - - >>> classification_two = PromptResponseClassification( - >>> class_type = PromptResponseClassification.Type.RESPONSE_RADIO, - >>> name = "Second Example") - - >>> classification_two.add_option(ResponseOption( - >>> value = "Option Example")) - - Attributes: - class_type: (Classification.Type) - name: (str) - instructions: (str) - required: (bool) - options: (list) - character_min: (int) - character_max: (int) - schema_id: (str) - feature_schema_id: (str) - """ - - def __post_init__(self): - if self.name is None: - msg = ( - "When creating the Classification feature, please use “name” " - "for the classification schema name, which will be used when " - "creating annotation payload for Model-Assisted Labeling " - "Import and Label Import. “instructions” is no longer " - "supported to specify classification schema name." - ) - if self.instructions is not None: - self.name = self.instructions - warnings.warn(msg) - else: - raise ValueError(msg) - else: - if self.instructions is None: - self.instructions = self.name - - class Type(Enum): - PROMPT = "prompt" - RESPONSE_TEXT = "response-text" - RESPONSE_CHECKLIST = "response-checklist" - RESPONSE_RADIO = "response-radio" - - _REQUIRES_OPTIONS = {Type.RESPONSE_CHECKLIST, Type.RESPONSE_RADIO} - - class_type: Type - name: Optional[str] = None - instructions: Optional[str] = None - required: bool = True - options: List[ResponseOption] = field(default_factory=list) - character_min: Optional[int] = None - character_max: Optional[int] = None - schema_id: Optional[str] = None - feature_schema_id: Optional[str] = None - - @classmethod - def from_dict(cls, dictionary: Dict[str, Any]) -> Dict[str, Any]: - return cls( - class_type=PromptResponseClassification.Type(dictionary["type"]), - name=dictionary["name"], - instructions=dictionary["instructions"], - required=True, # always required - options=[ - ResponseOption.from_dict(o) for o in dictionary["options"] - ], - character_min=dictionary.get("minCharacters", None), - character_max=dictionary.get("maxCharacters", None), - schema_id=dictionary.get("schemaNodeId", None), - feature_schema_id=dictionary.get("featureSchemaId", None), - ) - - def asdict(self, is_subclass: bool = False) -> Dict[str, Any]: - if self.class_type in self._REQUIRES_OPTIONS and len(self.options) < 1: - raise InconsistentOntologyException( - f"Response Classification '{self.name}' requires options." - ) - classification = { - "type": self.class_type.value, - "instructions": self.instructions, - "name": self.name, - "required": True, # always required - "options": [o.asdict() for o in self.options], - "schemaNodeId": self.schema_id, - "featureSchemaId": self.feature_schema_id, - } - if ( - self.class_type == self.Type.PROMPT - or self.class_type == self.Type.RESPONSE_TEXT - ): - if self.character_min: - classification["minCharacters"] = self.character_min - if self.character_max: - classification["maxCharacters"] = self.character_max - if is_subclass: - return classification - return classification - - def add_option(self, option: ResponseOption) -> None: - if option.value in (o.value for o in self.options): - raise InconsistentOntologyException( - f"Duplicate option '{option.value}' " - f"for response classification '{self.name}'." - ) - self.options.append(option) - - @dataclass class Tool: """ @@ -606,9 +245,9 @@ class OntologyBuilder: """ - tools: List[Union[Tool, StepReasoningTool, FactCheckingTool]] = field( - default_factory=list - ) + tools: List[ + Union[Tool, StepReasoningTool, FactCheckingTool, PromptIssueTool] + ] = field(default_factory=list) classifications: List[ Union[Classification, PromptResponseClassification] ] = field(default_factory=list) diff --git a/libs/labelbox/src/labelbox/schema/tool_building/__init__.py b/libs/labelbox/src/labelbox/schema/tool_building/__init__.py index dab325388..5a555fdff 100644 --- a/libs/labelbox/src/labelbox/schema/tool_building/__init__.py +++ b/libs/labelbox/src/labelbox/schema/tool_building/__init__.py @@ -1,4 +1,7 @@ import labelbox.schema.tool_building.tool_type import labelbox.schema.tool_building.step_reasoning_tool import labelbox.schema.tool_building.fact_checking_tool +import labelbox.schema.tool_building.prompt_issue_tool import labelbox.schema.tool_building.tool_type_mapping +import labelbox.schema.tool_building.types +import labelbox.schema.tool_building.classification diff --git a/libs/labelbox/src/labelbox/schema/tool_building/base_step_reasoning_tool.py b/libs/labelbox/src/labelbox/schema/tool_building/base_step_reasoning_tool.py index c9c9809d0..624e951a7 100644 --- a/libs/labelbox/src/labelbox/schema/tool_building/base_step_reasoning_tool.py +++ b/libs/labelbox/src/labelbox/schema/tool_building/base_step_reasoning_tool.py @@ -79,10 +79,6 @@ class _BaseStepReasoningTool(ABC): required: bool = False def __post_init__(self): - warnings.warn( - "This feature is experimental and subject to change.", - ) - if not self.name.strip(): raise ValueError("Name is required") diff --git a/libs/labelbox/src/labelbox/schema/tool_building/classification.py b/libs/labelbox/src/labelbox/schema/tool_building/classification.py new file mode 100644 index 000000000..9c0c69bea --- /dev/null +++ b/libs/labelbox/src/labelbox/schema/tool_building/classification.py @@ -0,0 +1,367 @@ +import warnings +from dataclasses import dataclass, field +from enum import Enum +from typing import Any, Dict, List, Optional, Union + +from lbox.exceptions import InconsistentOntologyException + +from labelbox.schema.tool_building.types import FeatureSchemaId + + +@dataclass +class Classification: + """ + A classification to be added to a Project's ontology. The + classification is dependent on the Classification Type. + + To instantiate, the "class_type" and "name" parameters must + be passed in. + + The "options" parameter holds a list of Option objects. This is not + necessary for some Classification types, such as TEXT. To see which + types require options, look at the "_REQUIRES_OPTIONS" class variable. + + Example(s): + classification = Classification( + class_type = Classification.Type.TEXT, + name = "Classification Example") + + classification_two = Classification( + class_type = Classification.Type.RADIO, + name = "Second Example") + classification_two.add_option(Option( + value = "Option Example")) + + Attributes: + class_type: (Classification.Type) + name: (str) + instructions: (str) + required: (bool) + options: (list) + ui_mode: (str) + schema_id: (str) + feature_schema_id: (str) + scope: (str) + """ + + class Type(Enum): + TEXT = "text" + CHECKLIST = "checklist" + RADIO = "radio" + + class Scope(Enum): + GLOBAL = "global" + INDEX = "index" + + class UIMode(Enum): + HOTKEY = "hotkey" + SEARCHABLE = "searchable" + + _REQUIRES_OPTIONS = {Type.CHECKLIST, Type.RADIO} + + class_type: Type + name: Optional[str] = None + instructions: Optional[str] = None + required: bool = False + options: List["Option"] = field(default_factory=list) + schema_id: Optional[str] = None + feature_schema_id: Optional[str] = None + scope: Optional[Scope] = None + ui_mode: Optional[UIMode] = ( + None # How this classification should be answered (e.g. hotkeys / autocomplete, etc) + ) + + def __post_init__(self): + if self.name is None: + msg = ( + "When creating the Classification feature, please use “name” " + "for the classification schema name, which will be used when " + "creating annotation payload for Model-Assisted Labeling " + "Import and Label Import. “instructions” is no longer " + "supported to specify classification schema name." + ) + if self.instructions is not None: + self.name = self.instructions + warnings.warn(msg) + else: + raise ValueError(msg) + else: + if self.instructions is None: + self.instructions = self.name + + @classmethod + def from_dict(cls, dictionary: Dict[str, Any]) -> "Classification": + return cls( + class_type=Classification.Type(dictionary["type"]), + name=dictionary["name"], + instructions=dictionary["instructions"], + required=dictionary.get("required", False), + options=[Option.from_dict(o) for o in dictionary["options"]], + ui_mode=cls.UIMode(dictionary["uiMode"]) + if "uiMode" in dictionary + else None, + schema_id=dictionary.get("schemaNodeId", None), + feature_schema_id=dictionary.get("featureSchemaId", None), + scope=cls.Scope(dictionary.get("scope", cls.Scope.GLOBAL)), + ) + + def asdict(self, is_subclass: bool = False) -> Dict[str, Any]: + if self.class_type in self._REQUIRES_OPTIONS and len(self.options) < 1: + raise InconsistentOntologyException( + f"Classification '{self.name}' requires options." + ) + classification = { + "type": self.class_type.value, + "instructions": self.instructions, + "name": self.name, + "required": self.required, + "options": [o.asdict() for o in self.options], + "schemaNodeId": self.schema_id, + "featureSchemaId": self.feature_schema_id, + } + if ( + self.class_type == self.Type.RADIO + or self.class_type == self.Type.CHECKLIST + ) and self.ui_mode: + # added because this key does nothing for text so no point of including + classification["uiMode"] = self.ui_mode.value + if is_subclass: + return classification + classification["scope"] = ( + self.scope.value + if self.scope is not None + else self.Scope.GLOBAL.value + ) + return classification + + def add_option(self, option: "Option") -> None: + if option.value in (o.value for o in self.options): + raise InconsistentOntologyException( + f"Duplicate option '{option.value}' " + f"for classification '{self.name}'." + ) + self.options.append(option) + + +@dataclass +class Option: + """ + An option is a possible answer within a Classification object in + a Project's ontology. + + To instantiate, only the "value" parameter needs to be passed in. + + Example(s): + option = Option(value = "Option Example") + + Attributes: + value: (str) + schema_id: (str) + feature_schema_id: (str) + options: (list) + """ + + value: Union[str, int] + label: Optional[Union[str, int]] = None + schema_id: Optional[str] = None + feature_schema_id: Optional[FeatureSchemaId] = None # type: ignore + options: Union[ + List["Classification"], List["PromptResponseClassification"] + ] = field(default_factory=list) + + def __post_init__(self): + if self.label is None: + self.label = self.value + + @classmethod + def from_dict(cls, dictionary: Dict[str, Any]) -> "Option": + return cls( + value=dictionary["value"], + label=dictionary["label"], + schema_id=dictionary.get("schemaNodeId", None), + feature_schema_id=dictionary.get("featureSchemaId", None), + options=[ + Classification.from_dict(o) + for o in dictionary.get("options", []) + ], + ) + + def asdict(self) -> Dict[str, Any]: + return { + "schemaNodeId": self.schema_id, + "featureSchemaId": self.feature_schema_id, + "label": self.label, + "value": self.value, + "options": [o.asdict(is_subclass=True) for o in self.options], + } + + def add_option( + self, option: Union["Classification", "PromptResponseClassification"] + ) -> None: + if option.name in (o.name for o in self.options): + raise InconsistentOntologyException( + f"Duplicate nested classification '{option.name}' " + f"for option '{self.label}'" + ) + self.options.append(option) # type: ignore + + +@dataclass +class PromptResponseClassification: + """ + + A PromptResponseClassification to be added to a Project's ontology. The + classification is dependent on the PromptResponseClassification Type. + + To instantiate, the "class_type" and "name" parameters must + be passed in. + + The "options" parameter holds a list of Response Option objects. This is not + necessary for some Classification types, such as RESPONSE_TEXT or PROMPT. To see which + types require options, look at the "_REQUIRES_OPTIONS" class variable. + + Example(s): + >>> classification = PromptResponseClassification( + >>> class_type = PromptResponseClassification.Type.Prompt, + >>> character_min = 1, + >>> character_max = 1 + >>> name = "Prompt Classification Example") + + >>> classification_two = PromptResponseClassification( + >>> class_type = PromptResponseClassification.Type.RESPONSE_RADIO, + >>> name = "Second Example") + + >>> classification_two.add_option(ResponseOption( + >>> value = "Option Example")) + + Attributes: + class_type: (Classification.Type) + name: (str) + instructions: (str) + required: (bool) + options: (list) + character_min: (int) + character_max: (int) + schema_id: (str) + feature_schema_id: (str) + """ + + def __post_init__(self): + if self.name is None: + msg = ( + "When creating the Classification feature, please use “name” " + "for the classification schema name, which will be used when " + "creating annotation payload for Model-Assisted Labeling " + "Import and Label Import. “instructions” is no longer " + "supported to specify classification schema name." + ) + if self.instructions is not None: + self.name = self.instructions + warnings.warn(msg) + else: + raise ValueError(msg) + else: + if self.instructions is None: + self.instructions = self.name + + class Type(Enum): + PROMPT = "prompt" + RESPONSE_TEXT = "response-text" + RESPONSE_CHECKLIST = "response-checklist" + RESPONSE_RADIO = "response-radio" + + _REQUIRES_OPTIONS = {Type.RESPONSE_CHECKLIST, Type.RESPONSE_RADIO} + + class_type: Type + name: Optional[str] = None + instructions: Optional[str] = None + required: bool = True + options: List["ResponseOption"] = field(default_factory=list) + character_min: Optional[int] = None + character_max: Optional[int] = None + schema_id: Optional[str] = None + feature_schema_id: Optional[str] = None + + @classmethod + def from_dict( + cls, dictionary: Dict[str, Any] + ) -> "PromptResponseClassification": + return cls( + class_type=PromptResponseClassification.Type(dictionary["type"]), + name=dictionary["name"], + instructions=dictionary["instructions"], + required=True, # always required + options=[ + ResponseOption.from_dict(o) for o in dictionary["options"] + ], + character_min=dictionary.get("minCharacters", None), + character_max=dictionary.get("maxCharacters", None), + schema_id=dictionary.get("schemaNodeId", None), + feature_schema_id=dictionary.get("featureSchemaId", None), + ) + + def asdict(self, is_subclass: bool = False) -> Dict[str, Any]: + if self.class_type in self._REQUIRES_OPTIONS and len(self.options) < 1: + raise InconsistentOntologyException( + f"Response Classification '{self.name}' requires options." + ) + classification: Dict[str, Any] = { + "type": self.class_type.value, + "instructions": self.instructions, + "name": self.name, + "required": True, # always required + "options": [o.asdict() for o in self.options], + "schemaNodeId": self.schema_id, + "featureSchemaId": self.feature_schema_id, + } + if ( + self.class_type == self.Type.PROMPT + or self.class_type == self.Type.RESPONSE_TEXT + ): + if self.character_min: + classification["minCharacters"] = self.character_min + if self.character_max: + classification["maxCharacters"] = self.character_max + if is_subclass: + return classification + return classification + + def add_option(self, option: "ResponseOption") -> None: + if option.value in (o.value for o in self.options): + raise InconsistentOntologyException( + f"Duplicate option '{option.value}' " + f"for response classification '{self.name}'." + ) + self.options.append(option) + + +@dataclass +class ResponseOption(Option): + """ + An option is a possible answer within a PromptResponseClassification response object in + a Project's ontology. + + To instantiate, only the "value" parameter needs to be passed in. + + Example(s): + option = ResponseOption(value = "Response Option Example") + + Attributes: + value: (str) + schema_id: (str) + feature_schema_id: (str) + options: (list) + """ + + @classmethod + def from_dict(cls, dictionary: Dict[str, Any]) -> "ResponseOption": + return cls( + value=dictionary["value"], + label=dictionary["label"], + schema_id=dictionary.get("schemaNodeId", None), + feature_schema_id=dictionary.get("featureSchemaId", None), + options=[ + PromptResponseClassification.from_dict(o) + for o in dictionary.get("options", []) + ], + ) diff --git a/libs/labelbox/src/labelbox/schema/tool_building/fact_checking_tool.py b/libs/labelbox/src/labelbox/schema/tool_building/fact_checking_tool.py index 4a02a482c..440f343cd 100644 --- a/libs/labelbox/src/labelbox/schema/tool_building/fact_checking_tool.py +++ b/libs/labelbox/src/labelbox/schema/tool_building/fact_checking_tool.py @@ -65,6 +65,8 @@ def build_fact_checking_definition(): class FactCheckingTool(_BaseStepReasoningTool): """ Use this class in OntologyBuilder to create a tool for fact checking + + Note variant kinds can not be changed """ type: ToolType = field(default=ToolType.FACT_CHECKING, init=False) diff --git a/libs/labelbox/src/labelbox/schema/tool_building/prompt_issue_tool.py b/libs/labelbox/src/labelbox/schema/tool_building/prompt_issue_tool.py new file mode 100644 index 000000000..5b9b55d9c --- /dev/null +++ b/libs/labelbox/src/labelbox/schema/tool_building/prompt_issue_tool.py @@ -0,0 +1,97 @@ +from dataclasses import dataclass, field +from typing import Any, Dict, List, Optional + +from labelbox.schema.tool_building.classification import ( + Classification, + Option, +) +from labelbox.schema.tool_building.tool_type import ToolType + + +def _supported_classifications() -> List[Classification]: + option_1_text = "This prompt cannot be rated (eg. contains PII, a nonsense prompt, a foreign language, or other scenario that makes the responses impossible to assess reliably). However, if you simply do not have expertise to tackle this prompt, please skip the task; do not mark it as not rateable." + option_2_text = "This prompt contains a false, offensive, or controversial premise (eg. “why does 1+1=3”?)" + option_3_text = "This prompt is not self-contained, i.e. the prompt cannot be understood without additional context about previous turns, account information or images." + options = [ + Option(label=option_1_text, value="not_rateable"), + Option(label=option_2_text, value="false_offensive_controversial"), + Option(label=option_3_text, value="not_self_contained"), + ] + return [ + Classification( + class_type=Classification.Type.CHECKLIST, + name="prompt_issue", + options=options, + ), + ] + + +@dataclass +class PromptIssueTool: + """ + Use this class in OntologyBuilder to create a tool for prompt rating + It comes with a prebuild checklist of options, which a user can modify or override + So essentially this is a tool with a prebuilt checklist classification + """ + + name: str + type: ToolType = field(default=ToolType.PROMPT_ISSUE, init=False) + required: bool = False + schema_id: Optional[str] = None + feature_schema_id: Optional[str] = None + color: Optional[str] = None + classifications: List[Classification] = field( + default_factory=_supported_classifications + ) + + def __post_init__(self): + if self.name.strip() == "": + raise ValueError("Name cannot be empty") + + if not self._validate_classifications(self.classifications): + raise ValueError("Only one checklist classification is supported") + + def __setattr__(self, name, value): + if name == "classifications" and not self._validate_classifications( + value + ): + raise ValueError("Classifications are immutable") + object.__setattr__(self, name, value) + + def _validate_classifications( + self, classifications: List[Classification] + ) -> bool: + if ( + len(classifications) != 1 + or classifications[0].class_type != Classification.Type.CHECKLIST + ): + return False + return True + + def asdict(self) -> Dict[str, Any]: + return { + "tool": self.type.value, + "name": self.name, + "required": self.required, + "schemaNodeId": self.schema_id, + "featureSchemaId": self.feature_schema_id, + "classifications": [ + classification.asdict() + for classification in self.classifications + ], + "color": self.color, + } + + @classmethod + def from_dict(cls, dictionary: Dict[str, Any]) -> "PromptIssueTool": + return cls( + name=dictionary["name"], + schema_id=dictionary.get("schemaNodeId", None), + feature_schema_id=dictionary.get("featureSchemaId", None), + required=dictionary.get("required", False), + classifications=[ + Classification.from_dict(classification) + for classification in dictionary["classifications"] + ], + color=dictionary.get("color", None), + ) diff --git a/libs/labelbox/src/labelbox/schema/tool_building/tool_type.py b/libs/labelbox/src/labelbox/schema/tool_building/tool_type.py index faab626ff..9fc0ba892 100644 --- a/libs/labelbox/src/labelbox/schema/tool_building/tool_type.py +++ b/libs/labelbox/src/labelbox/schema/tool_building/tool_type.py @@ -4,6 +4,7 @@ class ToolType(Enum): STEP_REASONING = "step-reasoning" FACT_CHECKING = "fact-checking" + PROMPT_ISSUE = "prompt-issue" @classmethod def valid(cls, tool_type: str) -> bool: diff --git a/libs/labelbox/src/labelbox/schema/tool_building/tool_type_mapping.py b/libs/labelbox/src/labelbox/schema/tool_building/tool_type_mapping.py index 68bfb4890..6de255cc3 100644 --- a/libs/labelbox/src/labelbox/schema/tool_building/tool_type_mapping.py +++ b/libs/labelbox/src/labelbox/schema/tool_building/tool_type_mapping.py @@ -1,5 +1,10 @@ -from labelbox.schema.tool_building.fact_checking_tool import FactCheckingTool -from labelbox.schema.tool_building.step_reasoning_tool import StepReasoningTool +from labelbox.schema.tool_building.fact_checking_tool import ( + FactCheckingTool, +) +from labelbox.schema.tool_building.prompt_issue_tool import PromptIssueTool +from labelbox.schema.tool_building.step_reasoning_tool import ( + StepReasoningTool, +) from labelbox.schema.tool_building.tool_type import ToolType @@ -12,3 +17,5 @@ def map_tool_type_to_tool_cls(tool_type_str: str): return StepReasoningTool elif tool_type == ToolType.FACT_CHECKING: return FactCheckingTool + elif tool_type == ToolType.PROMPT_ISSUE: + return PromptIssueTool diff --git a/libs/labelbox/src/labelbox/schema/tool_building/types.py b/libs/labelbox/src/labelbox/schema/tool_building/types.py new file mode 100644 index 000000000..0d6e34717 --- /dev/null +++ b/libs/labelbox/src/labelbox/schema/tool_building/types.py @@ -0,0 +1,6 @@ +from typing import Annotated + +from pydantic import Field + +FeatureSchemaId = Annotated[str, Field(min_length=25, max_length=25)] +SchemaId = Annotated[str, Field(min_length=25, max_length=25)] diff --git a/libs/labelbox/tests/integration/conftest.py b/libs/labelbox/tests/integration/conftest.py index acc400c21..11984cbd7 100644 --- a/libs/labelbox/tests/integration/conftest.py +++ b/libs/labelbox/tests/integration/conftest.py @@ -16,6 +16,7 @@ MediaType, OntologyBuilder, Option, + PromptIssueTool, PromptResponseClassification, ResponseOption, StepReasoningTool, @@ -581,6 +582,7 @@ def chat_evaluation_ontology(client, rand_gen): ), StepReasoningTool(name="step reasoning"), FactCheckingTool(name="fact checking"), + PromptIssueTool(name="prompt issue"), ], classifications=[ Classification( diff --git a/libs/labelbox/tests/integration/test_chat_evaluation_ontology_project.py b/libs/labelbox/tests/integration/test_chat_evaluation_ontology_project.py index c5db9760c..8d94bdba2 100644 --- a/libs/labelbox/tests/integration/test_chat_evaluation_ontology_project.py +++ b/libs/labelbox/tests/integration/test_chat_evaluation_ontology_project.py @@ -15,7 +15,7 @@ def test_create_chat_evaluation_ontology_project( # here we are essentially testing the ontology creation which is a fixture assert ontology assert ontology.name - assert len(ontology.tools()) == 5 + assert len(ontology.tools()) == 6 for tool in ontology.tools(): assert tool.schema_id assert tool.feature_schema_id diff --git a/libs/labelbox/tests/integration/test_ontology.py b/libs/labelbox/tests/integration/test_ontology.py index f87197b62..febfdacfd 100644 --- a/libs/labelbox/tests/integration/test_ontology.py +++ b/libs/labelbox/tests/integration/test_ontology.py @@ -5,8 +5,15 @@ from labelbox import MediaType, OntologyBuilder, Tool from labelbox.orm.model import Entity -from labelbox.schema.tool_building.fact_checking_tool import FactCheckingTool -from labelbox.schema.tool_building.step_reasoning_tool import StepReasoningTool +from labelbox.schema.tool_building.classification import Classification +from labelbox.schema.tool_building.fact_checking_tool import ( + FactCheckingTool, +) +from labelbox.schema.tool_building.prompt_issue_tool import PromptIssueTool +from labelbox.schema.tool_building.step_reasoning_tool import ( + StepReasoningTool, +) +from labelbox.schema.tool_building.tool_type import ToolType def test_feature_schema_is_not_archived(client, ontology): @@ -444,3 +451,40 @@ def test_fact_checking_ontology(chat_evaluation_ontology): ], "version": 1, } + + +def test_prompt_issue_ontology(chat_evaluation_ontology): + ontology = chat_evaluation_ontology + prompt_issue = None + for tool in ontology.normalized["tools"]: + if tool["tool"] == "prompt-issue": + prompt_issue = tool + break + assert prompt_issue is not None + + assert prompt_issue["definition"] == { + "title": "prompt issue", + "value": "prompt_issue", + "color": "#ff00ff", + } + assert prompt_issue["schemaNodeId"] is not None + assert prompt_issue["featureSchemaId"] is not None + assert len(prompt_issue["classifications"]) == 1 + + prompt_issue_tool = None + for tool in ontology.tools(): + if isinstance(tool, PromptIssueTool): + prompt_issue_tool = tool + break + assert prompt_issue_tool is not None + # Assertions + assert prompt_issue_tool.name == "prompt issue" + assert prompt_issue_tool.type == ToolType.PROMPT_ISSUE + assert prompt_issue_tool.schema_id is not None + assert prompt_issue_tool.feature_schema_id is not None + + # Check classifications + assert len(prompt_issue_tool.classifications) == 1 + classification = prompt_issue_tool.classifications[0] + assert classification.class_type == Classification.Type.CHECKLIST + assert len(classification.options) == 3 # Check number of options diff --git a/libs/labelbox/tests/unit/test_unit_fact_checking_tool.py b/libs/labelbox/tests/unit/test_unit_fact_checking_tool.py index 019f1355b..6f0739d30 100644 --- a/libs/labelbox/tests/unit/test_unit_fact_checking_tool.py +++ b/libs/labelbox/tests/unit/test_unit_fact_checking_tool.py @@ -41,3 +41,47 @@ def test_fact_checking_as_dict_default(): } assert tool_dict == expected_dict + + +def test_step_reasoning_as_dict_with_actions(): + tool = FactCheckingTool(name="Fact Checking Tool") + for variant in tool.definition.variants: + variant.set_actions([]) + + # Get the dictionary representation + tool_dict = tool.asdict() + + # Expected dictionary structure + expected_dict = { + "tool": "fact-checking", + "name": "Fact Checking Tool", + "required": False, + "schemaNodeId": None, + "featureSchemaId": None, + "color": None, + "definition": { + "variants": [ + {"id": 0, "name": "Accurate", "actions": []}, + {"id": 1, "name": "Inaccurate", "actions": []}, + {"id": 2, "name": "Disputed", "actions": []}, + { + "id": 3, + "name": "Unsupported", + "actions": [], + }, + { + "id": 4, + "name": "Can't confidently assess", + "actions": [], + }, + { + "id": 5, + "name": "No factual information", + "actions": [], + }, + ], + "version": 1, + }, + } + + assert tool_dict == expected_dict diff --git a/libs/labelbox/tests/unit/test_unit_prompt_issue_tool.py b/libs/labelbox/tests/unit/test_unit_prompt_issue_tool.py new file mode 100644 index 000000000..04dc89668 --- /dev/null +++ b/libs/labelbox/tests/unit/test_unit_prompt_issue_tool.py @@ -0,0 +1,70 @@ +import pytest + +from labelbox.schema.tool_building.classification import Classification +from labelbox.schema.tool_building.prompt_issue_tool import PromptIssueTool + + +def test_as_dict(): + tool = PromptIssueTool(name="Prompt Issue Tool") + + # Get the dictionary representation + tool_dict = tool.asdict() + expected_dict = { + "tool": "prompt-issue", + "name": "Prompt Issue Tool", + "required": False, + "schemaNodeId": None, + "featureSchemaId": None, + "classifications": [ + { + "type": "checklist", + "instructions": "prompt_issue", + "name": "prompt_issue", + "required": False, + "options": [ + { + "schemaNodeId": None, + "featureSchemaId": None, + "label": "This prompt cannot be rated (eg. contains PII, a nonsense prompt, a foreign language, or other scenario that makes the responses impossible to assess reliably). However, if you simply do not have expertise to tackle this prompt, please skip the task; do not mark it as not rateable.", + "value": "not_rateable", + "options": [], + }, + { + "schemaNodeId": None, + "featureSchemaId": None, + "label": "This prompt contains a false, offensive, or controversial premise (eg. “why does 1+1=3”?)", + "value": "false_offensive_controversial", + "options": [], + }, + { + "schemaNodeId": None, + "featureSchemaId": None, + "label": "This prompt is not self-contained, i.e. the prompt cannot be understood without additional context about previous turns, account information or images.", + "value": "not_self_contained", + "options": [], + }, + ], + "schemaNodeId": None, + "featureSchemaId": None, + "scope": "global", + } + ], + "color": None, + } + assert tool_dict == expected_dict + + with pytest.raises(ValueError): + tool.classifications = [ + Classification(Classification.Type.TEXT, "prompt_issue") + ] + + with pytest.raises(ValueError): + tool.classifications = "abcd" + + # Test that we can modify the classification options + classification = tool.classifications[0] + classification.options.pop() + classification.options[0].label = "test" + classification.options[0].value == "test_value" + tool.classifications = [classification] + assert tool.classifications == [classification]