[data] [docs] Adding unstructured data templates from ray summit 2025

.vale/styles/config/vocabularies/Data/accept.txt

@@ -1,13 +1,15 @@
+assess_quality
 autoscaler
 [Aa]vro
 [Bb]ackpressure
 Dask
 [Dd]ata('s)?
 [Dd]atasource(s)?
 [Dd]iscretizer(s)?
+docstrings
 dtype
 FLAC
-[Gg]roupby
+[Gg]roup[bB]y
 [Hh]asher(s)?
 [Hh]udi
 [Ii]ndexable
@@ -19,14 +21,18 @@ MCAP
 Modin
 [Mm]ultiget(s)?
 ndarray(s)?
+NLP
 [Oo]utqueue(s)?
+PDFs
 [Pp]ipelined
 Predibase('s)?
 [Pp]refetch
 [Pp]refetching
 [Pp]reprocess
 [Pp]reprocessor(s)?
+process_file
 [Pp]ushdown
+queryable
 RGB
 runai
 [Ss]calers
@@ -36,5 +42,3 @@ UDF(s)?
 VLM(s)?
 XGBoost
 YOLO
-[Ss]harding
-[Ss]harded

doc/BUILD.bazel

@@ -700,9 +700,15 @@ doctest(
     env = {"RAY_TRAIN_V2_ENABLED": "1"},
 )
 
+# --------------------------------------------------------------------
+# Discover the Anyscale Jobs compute configs .yaml for release tests in CI
+# --------------------------------------------------------------------
+
 filegroup(
     name = "example_configs",
-    srcs = glob(["source/ray-overview/examples/**/*.yaml"]),
+    srcs = glob([
+        "source/ray-overview/examples/**/*.yaml"
+    ]),
     visibility = ["//release:__pkg__"],
 )
 

doc/source/conf.py

@@ -235,6 +235,7 @@ def __init__(self, version: str):
     "train/examples/**/README.md",
     "serve/tutorials/deployment-serve-llm/README.*",
     "serve/tutorials/deployment-serve-llm/*/notebook.ipynb",
+    "data/examples/**/content/README.md",
     "ray-overview/examples/llamafactory-llm-fine-tune/README.ipynb",
     "ray-overview/examples/llamafactory-llm-fine-tune/**/*.ipynb",
 ] + autogen_files

doc/source/custom_directives.py

@@ -452,6 +452,10 @@ class UseCase(ExampleEnum):
     GENERATIVE_AI = "Generative AI"
     COMPUTER_VISION = "Computer Vision"
     NATURAL_LANGUAGE_PROCESSING = "Natural Language Processing"
+    ETL = "ETL"
+    DATA_INGESTION = "Data Ingestion"
+    DATA_WAREHOUSING = "Data Warehousing"
+    DOCUMENT_PROCESSING = "Document Processing"
 
     @classmethod
     def formatted_name(cls):
@@ -493,7 +497,9 @@ class Framework(ExampleEnum):
     HUGGINGFACE = "Hugging Face"
     DATAJUICER = "Data-Juicer"
     VLLM = "vLLM"
+    PANDAS = "Pandas"
     ANY = "Any"
+    UNSTRUCTURED = "Unstructured"
 
     @classmethod
     def formatted_name(cls):

doc/source/data/examples.yml

@@ -45,3 +45,12 @@ examples:
     frameworks:
       - xgboost
     link: ../train/examples/xgboost/distributed-xgboost-lightgbm
+  - title: Unstructured Data Ingestion and Processing
+    skill_level: advanced
+    frameworks:
+      - Transformers
+      - Unstructured
+    use_cases:
+      - document processing
+      - data ingestion
+    link: examples/unstructured-data-ingestion/content/unstructured-data-ingestion

doc/source/data/examples/BUILD.bazel

@@ -17,3 +17,12 @@ py_test_run_all_notebooks(
         data = ["//doc/source/data/examples:data_examples"],
     tags = ["exclusive", "team:data", "gpu"],
 )
+
+filegroup(
+    name = "data_example_configs",
+    srcs = glob([
+        "**/ci/aws.yaml",
+        "**/ci/gce.yaml"
+    ]),
+    visibility = ["//release:__pkg__"],
+)

doc/source/data/examples/unstructured-data-ingestion/ci/aws.yaml

@@ -0,0 +1,17 @@
+cloud_id: {{env["ANYSCALE_CLOUD_ID"]}}
+region: us-west-2
+
+head_node_type:
+  name: head
+  instance_type: m5.2xlarge
+  resources:
+    cpu: 0
+    gpu: 0
+worker_node_types:
+  - name: 8CPU-32GB
+    instance_type: m5.2xlarge
+    min_workers: 10
+    max_workers: 10
+
+flags:
+  allow-cross-zone-autoscaling: true

doc/source/data/examples/unstructured-data-ingestion/ci/build.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+set -exo pipefail
+
+# Install runtime deps
+pip install unstructured

doc/source/data/examples/unstructured-data-ingestion/ci/gce.yaml

@@ -0,0 +1,17 @@
+cloud_id: {{env["ANYSCALE_CLOUD_ID"]}}
+region: us-central1
+
+head_node_type:
+  name: head
+  instance_type: n1-standard-8
+  resources:
+    cpu: 0
+    gpu: 0
+worker_node_types:
+  - name: 8CPU-32GB
+    instance_type: n1-standard-8
+    min_workers: 10
+    max_workers: 10
+
+flags:
+  allow-cross-zone-autoscaling: true

doc/source/data/examples/unstructured-data-ingestion/ci/nb2py.py

@@ -0,0 +1,86 @@
+#!/usr/bin/env python3
+import argparse
+import nbformat
+
+
+def convert_notebook(
+    input_path: str, output_path: str, ignore_cmds: bool = False
+) -> None:
+    """
+    Read a Jupyter notebook and write a Python script, converting all %%bash
+    cells and IPython "!" commands into subprocess.run calls that raise on error.
+    Cells that load or autoreload extensions are ignored.
+    """
+    nb = nbformat.read(input_path, as_version=4)
+    with open(output_path, "w") as out:
+        for cell in nb.cells:
+            # Only process code cells
+            if cell.cell_type != "code":
+                continue
+
+            lines = cell.source.splitlines()
+            # Skip cells that load or autoreload extensions
+            if any(
+                l.strip().startswith("%load_ext autoreload")
+                or l.strip().startswith("%autoreload all")
+                for l in lines
+            ):
+                continue
+
+            # Detect a %%bash cell
+            if lines and lines[0].strip().startswith("%%bash"):
+                if ignore_cmds:
+                    continue
+                bash_script = "\n".join(lines[1:]).rstrip()
+                out.write("import subprocess\n")
+                out.write(
+                    f"subprocess.run(r'''{bash_script}''',\n"
+                    "               shell=True,\n"
+                    "               check=True,\n"
+                    "               executable='/bin/bash')\n\n"
+                )
+            else:
+                # Detect any IPython '!' shell commands in code lines
+                has_bang = any(line.lstrip().startswith("!") for line in lines)
+                if has_bang:
+                    if ignore_cmds:
+                        continue
+                    out.write("import subprocess\n")
+                    for line in lines:
+                        stripped = line.lstrip()
+                        if stripped.startswith("!"):
+                            cmd = stripped[1:].lstrip()
+                            out.write(
+                                f"subprocess.run(r'''{cmd}''',\n"
+                                "               shell=True,\n"
+                                "               check=True,\n"
+                                "               executable='/bin/bash')\n"
+                            )
+                        else:
+                            out.write(line.rstrip() + "\n")
+                    out.write("\n")
+                else:
+                    # Regular Python cell:
+                    code = cell.source.rstrip()
+                    # Example of filtering cells by content
+                    # if "client.chat.completions.create" in code:
+                    #    continue  # Model isn't deployed in CI so skip cells calling the service
+                    # else, dump as-is
+                    out.write(cell.source.rstrip() + "\n\n")
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(
+        description="Convert a Jupyter notebook to a Python script, preserving bash cells and '!' commands as subprocess calls unless ignored with --ignore-cmds."
+    )
+    parser.add_argument("input_nb", help="Path to the input .ipynb file")
+    parser.add_argument("output_py", help="Path for the output .py script")
+    parser.add_argument(
+        "--ignore-cmds", action="store_true", help="Ignore bash cells and '!' commands"
+    )
+    args = parser.parse_args()
+    convert_notebook(args.input_nb, args.output_py, ignore_cmds=args.ignore_cmds)
+
+
+if __name__ == "__main__":
+    main()

doc/source/data/examples/unstructured-data-ingestion/ci/tests.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+# Don't use nbconvert or jupytext unless you're willing
+# to check each subprocess unit and validate that errors
+# aren't being consumed/hidden
+
+set -exo pipefail
+
+# TODO once runnable on nightly, uncomment these lines to properly test
+python ci/nb2py.py content/unstructured-data-ingestion.ipynb unstructured-data-ingestion.py
+python unstructured-data-ingestion.py
+rm unstructured-data-ingestion.py

doc/source/data/examples/unstructured-data-ingestion/configs/aws.yaml

@@ -0,0 +1,14 @@
+head_node_type:
+  name: head
+  instance_type: m5.2xlarge
+  resources:
+    cpu: 0
+    gpu: 0
+worker_node_types:
+  - name: 8CPU-32GB
+    instance_type: m5.2xlarge
+    min_workers: 10
+    max_workers: 10
+
+flags:
+  allow-cross-zone-autoscaling: true

doc/source/data/examples/unstructured-data-ingestion/configs/gce.yaml

@@ -0,0 +1,14 @@
+head_node_type:
+  name: head
+  instance_type: n1-standard-8
+  resources:
+    cpu: 0
+    gpu: 0
+worker_node_types:
+  - name: 8CPU-32GB
+    instance_type: n1-standard-8
+    min_workers: 10
+    max_workers: 10
+
+flags:
+  allow-cross-zone-autoscaling: true