Table Enrichment - Descriptions and Indexing

Author: Shivani Kabu

docling_core/transforms/serializer/markdown.py

@@ -54,6 +54,7 @@
     TextItem,
     TitleItem,
     UnorderedList,
+    PictureDescriptionData
 )
 
 
@@ -62,7 +63,9 @@ class MarkdownParams(CommonParams):
 
     layers: set[ContentLayer] = {ContentLayer.BODY}
     image_mode: ImageRefMode = ImageRefMode.PLACEHOLDER
+    table_mode: ImageRefMode = ImageRefMode.PLACEHOLDER
     image_placeholder: str = "<!-- image -->"
+    table_placeholder: str =  "<!-- table -->"
     enable_chart_tables: bool = True
     indent: int = 4
     wrap_width: Optional[PositiveInt] = None
@@ -146,6 +149,7 @@ def serialize(
         **kwargs: Any,
     ) -> SerializationResult:
         """Serializes the passed item."""
+        params = MarkdownParams(**kwargs)
         res_parts: list[SerializationResult] = []
 
         cap_res = doc_serializer.serialize_captions(
@@ -156,6 +160,14 @@ def serialize(
             res_parts.append(cap_res)
 
         if item.self_ref not in doc_serializer.get_excluded_refs(**kwargs):
+            img_res = self._serialize_table_part(
+                item=item,
+                doc=doc,
+                table_mode=params.table_mode,
+                table_placeholder=params.table_placeholder,
+            )
+            if img_res.text:
+                res_parts.append(img_res)
             rows = [
                 [
                     # make sure that md tables are not broken
@@ -184,6 +196,41 @@ def serialize(
 
         return create_ser_result(text=text_res, span_source=res_parts)
 
+    def _serialize_table_part(
+    self,
+    item: TableItem,
+    doc: DoclingDocument,
+    table_mode: ImageRefMode,
+    table_placeholder: str,
+    **kwargs: Any,
+) -> SerializationResult:
+        error_response = (
+            "<!-- 🖼️❌ Image not available. "
+            "Please use `PdfPipelineOptions(generate_picture_images=True)`"
+            " -->"
+        )
+        if table_mode == ImageRefMode.PLACEHOLDER:
+            text_res = table_placeholder
+
+        elif table_mode == ImageRefMode.DESCRIPTION:
+            if isinstance(item.image, ImageRef):
+                table_description = ''
+                for annotation in item.annotations:
+                    if not isinstance(annotation, PictureDescriptionData):
+                        continue
+                    table_description+=annotation.text+'\n'
+
+                text = f"![Table]({item.self_ref})\n<Table_Description>\n{table_description}</Table_Description>"
+                text_res = text
+            else:
+                text_res = table_placeholder
+
+        else:
+            text_res = table_placeholder
+
+
+        return create_ser_result(text=text_res, span_source=item)
+
 
 class MarkdownPictureSerializer(BasePictureSerializer):
     """Markdown-specific picture item serializer."""
@@ -282,9 +329,24 @@ def _serialize_image_part(
                 text_res = image_placeholder
             else:
                 text_res = f"![Image]({str(item.image.uri)})"
+            
+        elif image_mode == ImageRefMode.DESCRIPTION:
+            if isinstance(item.image, ImageRef):
+                picture_description = ''
+                for annotation in item.annotations:
+                    if not isinstance(annotation, PictureDescriptionData):
+                        continue
+                    picture_description+=annotation.text+'\n'
+
+                text = f"![Image]({item.self_ref})\n<Image_Description>\n{picture_description}</Image_Description>"
+                text_res = text
+            else:
+                text_res = image_placeholder
+
         else:
             text_res = image_placeholder
 
+
         return create_ser_result(text=text_res, span_source=item)
 
 

docling_core/types/doc/document.py

@@ -1190,6 +1190,8 @@ class TableItem(FloatingItem):
         DocItemLabel.DOCUMENT_INDEX,
         DocItemLabel.TABLE,
     ] = DocItemLabel.TABLE
+    annotations: List[PictureDataType] = []
+
 
     def export_to_dataframe(self) -> pd.DataFrame:
         """Export the table as a Pandas DataFrame."""
@@ -1236,14 +1238,22 @@ def export_to_dataframe(self) -> pd.DataFrame:
 
         return df
 
-    def export_to_markdown(self, doc: Optional["DoclingDocument"] = None) -> str:
+    def export_to_markdown(
+            self, 
+            doc: Optional["DoclingDocument"] = None,
+            table_mode: ImageRefMode = ImageRefMode.DESCRIPTION,
+            table_placeholder: str = "<!-- table -->") -> str:
         """Export the table as markdown."""
         if doc is not None:
             from docling_core.transforms.serializer.markdown import (
-                MarkdownDocSerializer,
+                MarkdownDocSerializer, MarkdownParams
             )
 
-            serializer = MarkdownDocSerializer(doc=doc)
+            serializer = MarkdownDocSerializer(doc=doc,
+                                               params=MarkdownParams(
+                    table_mode=table_mode,
+                    table_placeholder=table_placeholder,
+            ))
             text = serializer.serialize(item=self).text
             return text
         else:
@@ -2261,6 +2271,7 @@ def add_text(
     def add_table(
         self,
         data: TableData,
+        annotations: List[PictureDataType] = [],
         caption: Optional[Union[TextItem, RefItem]] = None,  # This is not cool yet.
         prov: Optional[ProvenanceItem] = None,
         parent: Optional[NodeItem] = None,
@@ -2283,7 +2294,7 @@ def add_table(
         cref = f"#/tables/{table_index}"
 
         tbl_item = TableItem(
-            label=label, data=data, self_ref=cref, parent=parent.get_ref()
+            label=label, annotations=annotations, data=data, self_ref=cref, parent=parent.get_ref()
         )
         if prov:
             tbl_item.prov.append(prov)
@@ -2919,6 +2930,8 @@ def save_as_markdown(
         escaping_underscores: bool = True,
         image_placeholder: str = "<!-- image -->",
         image_mode: ImageRefMode = ImageRefMode.PLACEHOLDER,
+        table_placeholder: str = "<!-- table -->",
+        table_mode: ImageRefMode = ImageRefMode.PLACEHOLDER,
         indent: int = 4,
         text_width: int = -1,
         page_no: Optional[int] = None,
@@ -2946,6 +2959,8 @@ def save_as_markdown(
             escape_underscores=escaping_underscores,
             image_placeholder=image_placeholder,
             image_mode=image_mode,
+            table_placeholder=table_placeholder,
+            table_mode=table_mode,
             indent=indent,
             text_width=text_width,
             page_no=page_no,
@@ -2965,8 +2980,10 @@ def export_to_markdown(  # noqa: C901
         strict_text: bool = False,
         escape_underscores: bool = True,
         image_placeholder: str = "<!-- image -->",
+        table_placeholder: str = "<!-- table -->",
         enable_chart_tables: bool = True,
         image_mode: ImageRefMode = ImageRefMode.PLACEHOLDER,
+        table_mode: ImageRefMode = ImageRefMode.PLACEHOLDER,
         indent: int = 4,
         text_width: int = -1,
         page_no: Optional[int] = None,
@@ -3033,8 +3050,10 @@ def export_to_markdown(  # noqa: C901
                 stop_idx=to_element,
                 escape_underscores=escape_underscores,
                 image_placeholder=image_placeholder,
+                table_placeholder=table_placeholder,
                 enable_chart_tables=enable_chart_tables,
                 image_mode=image_mode,
+                table_mode=table_mode,
                 indent=indent,
                 wrap_width=text_width if text_width > 0 else None,
                 page_break_placeholder=page_break_placeholder,
@@ -4112,4 +4131,4 @@ def validate_document(cls, d: "DoclingDocument"):
         if not d.validate_tree(d.body) or not d.validate_tree(d.furniture):
             raise ValueError("Document hierachy is inconsistent.")
 
-        return d
+        return d