feat: add option to parallelize pdfs

.github/workflows/ci.yaml

@@ -19,22 +19,22 @@ jobs:
         python-version: ["3.9","3.10","3.11"]
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
     - name: Install dependencies
       run: make install-test
     - name: Run unit tests
       run: |
         pip install  .
-        make test
+        make test-unit
 
   lint:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Install dependencies
       run: pip install .[dev]
     - name: Lint

.gitignore

@@ -7,3 +7,5 @@ __pycache__/
 
 # human-added igore files
 .ipynb_checkpoints/
+.idea/
+overlay.yaml

.speakeasy/gen.lock

@@ -1,12 +1,12 @@
 lockVersion: 2.0.0
 id: 8b5fa338-9106-4734-abf0-e30d67044a90
 management:
-  docChecksum: 903444f359d1dfa6342c692ae3e5c7ff
+  docChecksum: a3f5635b2b400dcd048524def8dc3b1d
   docVersion: 0.0.1
   speakeasyVersion: internal
-  generationVersion: 2.250.19
-  releaseVersion: 0.18.0
-  configChecksum: 938a4a39baa5695a3140be3b858483d4
+  generationVersion: 2.252.7
+  releaseVersion: 0.18.2
+  configChecksum: 0031ef656a2630d61ebc3f5ae6b1e46e
   repoURL: https://github.com/Unstructured-IO/unstructured-python-client.git
   repoSubDirectory: .
   installationURL: https://github.com/Unstructured-IO/unstructured-python-client.git

Makefile

@@ -1,6 +1,7 @@
 PACKAGE_NAME := unstructured-python-client
 CURRENT_DIR := $(shell pwd)
 ARCH := $(shell uname -m)
+DOCKER_IMAGE ?= downloads.unstructured.io/unstructured-io/unstructured-api:latest
 
 ###########
 # Install #
@@ -25,14 +26,42 @@ install: install-test install-dev
 #################
 
 .PHONY: test
-test:
-	PYTHONPATH=. pytest \
-		_test_unstructured_client
+test: test-unit test-integration-docker
+
+.PHONY: test-unit
+test-unit:
+	PYTHONPATH=. pytest _test_unstructured_client -v -k "unit"
+
+# Assumes you have unstructured-api running on localhost:8000
+.PHONY: test-integration
+test-integration:
+	PYTHONPATH=. pytest _test_unstructured_client -v -k "integration"
+
+# Runs the unstructured-api in docker for tests
+.PHONY: test-integration-docker
+test-integration-docker:
+	-docker stop unstructured-api && docker kill unstructured-api
+	docker run --name unstructured-api -p 8000:8000 -d --rm ${DOCKER_IMAGE} --host 0.0.0.0 && \
+	curl -s -o /dev/null --retry 10 --retry-delay 5 --retry-all-errors http://localhost:8000/general/docs && \
+	PYTHONPATH=. pytest _test_unstructured_client -v -k "integration" && \
+	docker kill unstructured-api
 
 .PHONY: lint
 lint:
 	pylint --rcfile=pylintrc src
 
+#############
+# Speakeasy #
+#############
+
+.PHONY: client-generate
+client-generate:
+	speakeasy generate sdk -s ./openapi.json -o ./ -l python
+
+diff-openapi:
+	speakeasy overlay compare --schemas=./openapi.json --schemas=./openapi_backend.json > ./overlay.yaml
+
+
 ###########
 # Jupyter #
 ###########

README.md

@@ -36,7 +36,7 @@ from unstructured_client.models.errors import SDKError
 
 s = UnstructuredClient(api_key_auth="YOUR_API_KEY")
 
-filename = "sample-docs/layout-parser-paper-fast.pdf"
+filename = "_sample_docs/layout-parser-paper-fast.pdf"
 
 with open(filename, "rb") as f:
     # Note that this currently only supports a single file
@@ -50,6 +50,7 @@ req = shared.PartitionParameters(
     # Other partition params
     strategy='ocr_only',
     languages=["eng"],
+    split_pdf_page=True,
 )
 
 try:
@@ -114,15 +115,59 @@ s = unstructured_client.UnstructuredClient(client: http_client)
 
 <!-- Placeholder for Future Speakeasy SDK Sections -->
 
+### Parameters
+
+`PartitionParameters` has parameters listed in `docs/models/shared/partitionparameters.md`
+
+More details about:
+1. `split_pdf_page` - when you set to True, it will cause the pdf to be split at client side, 
+   before sending to api. This currently causes the `parent_id`  metadata generation in elements to be disabled, as that
+   requires having context of multiple pages. This will work only for pdf files, so don't set it for other filetypes. 
+   The amount of threads that will be used for sending individual pdf pages, is controlled by `UNSTRUCTURED_CLIENT_SPLIT_CALL_THREADS` env var.
+   By default it equals to 5. It can't be more than 15, to avoid too high resource usage and costs.
+
 ### Maturity
 
 This SDK is in beta, and there may be breaking changes between versions without a major version update. Therefore, we recommend pinning usage
 to a specific package version. This way, you can install the same version each time without breaking changes unless you are intentionally
 looking for the latest version.
 
+### Installation Instructions for Local Development
+
+The following instructions are intended to help you get up and running with `unstructured-python-client` locally if you are planning to contribute to the project.
+
+* Using `pyenv` to manage virtualenv's is recommended but not necessary
+   * Mac install instructions. See [here](https://github.com/Unstructured-IO/community#mac--homebrew) for more detailed instructions.
+      * `brew install pyenv-virtualenv`
+      * `pyenv install 3.10`
+   * Linux instructions are available [here](https://github.com/Unstructured-IO/community#linux).
+
+* Create a virtualenv to work in and activate it, e.g. for one named `unstructured-python-client`:
+
+  `pyenv  virtualenv 3.10 unstructured-python-client` <br />
+  `pyenv activate unstructured-python-client`
+
+* Run `make install` and `make test`
+
 ### Contributions
 
-While we value open-source contributions to this SDK, this library is generated programmatically.
+While we value open-source contributions to this SDK, this library is generated programmatically by Speakeasy. In order to start working with this repo, you need to:
+1. Install Speakeasy client locally https://github.com/speakeasy-api/speakeasy#installation
+2. Run `speakeasy auth login`
+3. Run `make client-generate`. This allows to iterate development with python client.
+
+There are two important files:
+1. `openapi_backend.json` is maintained to represent the API that is supported on backend.
+2. `openapi.json` is treated as a source of truth for what should be accepted in python client.
+
+The idea is, that it is easier to maintain this file showing exactly what we support, instead of handcrafting diffs.
+
+When `openapi_backend.json` and `openapi.json` are compared using `make diff-openapi`, the diff is created
+which forms Speakeasy overlay, which shows exactly where are the differences. This is useful for proof checking.
+
+Once PR with changes is merged, Github CI will autogenerate the Speakeasy client in a new PR.
+You will have to manually bring back the human created lines in it.
+
 Feel free to open a PR or a Github issue as a proof of concept and we'll do our best to include it in a future release!
 
 ### SDK Created by [Speakeasy](https://www.speakeasyapi.dev/docs/sdk-design/python/methodology-python)

_test_unstructured_client/test__decorators.py

@@ -1,9 +1,12 @@
+import os
+import pypdf
 import pytest
+import requests
+from deepdiff import DeepDiff
 
 from unstructured_client import UnstructuredClient
 from unstructured_client.models import shared
-from unstructured_client.models.errors import SDKError
-
+from unstructured_client.models.errors import SDKError, HTTPValidationError
 
 FAKE_KEY = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
 
@@ -20,7 +23,7 @@
         "unstructured-000mock.api.unstructuredapp.io/general/v0/general",
     ],
 )
-def test_clean_server_url_fixes_malformed_paid_api_url(server_url: str):
+def test_unit_clean_server_url_fixes_malformed_paid_api_url(server_url: str):
     client = UnstructuredClient(
         server_url=server_url,
         api_key_auth=FAKE_KEY,
@@ -42,20 +45,20 @@ def test_clean_server_url_fixes_malformed_paid_api_url(server_url: str):
         "http://localhost:8000/general/v0/general",
     ],
 )
-def test_clean_server_url_fixes_malformed_localhost_url(server_url: str):
+def test_unit_clean_server_url_fixes_malformed_localhost_url(server_url: str):
     client = UnstructuredClient(
         server_url=server_url,
         api_key_auth=FAKE_KEY,
     )
     assert client.general.sdk_configuration.server_url == "http://localhost:8000"
 
 
-def test_clean_server_url_returns_empty_string_given_empty_string():
+def test_unit_clean_server_url_returns_empty_string_given_empty_string():
     client = UnstructuredClient( server_url="", api_key_auth=FAKE_KEY)
     assert client.general.sdk_configuration.server_url == ""
 
 
-def test_clean_server_url_returns_None_given_no_server_url():
+def test_unit_clean_server_url_returns_None_given_no_server_url():
     client = UnstructuredClient(
         api_key_auth=FAKE_KEY,
     )
@@ -71,7 +74,7 @@ def test_clean_server_url_returns_None_given_no_server_url():
         "unstructured-000mock.api.unstructuredapp.io/general/v0/general",
     ],
 )
-def test_clean_server_url_fixes_malformed_urls_with_positional_arguments(
+def test_unit_clean_server_url_fixes_malformed_urls_with_positional_arguments(
     server_url: str,
 ):
     client = UnstructuredClient(
@@ -85,7 +88,7 @@ def test_clean_server_url_fixes_malformed_urls_with_positional_arguments(
     )
 
 
-def test_suggest_defining_url_issues_a_warning_on_a_401():
+def test_unit_suggest_defining_url_issues_a_warning_on_a_401():
     client = UnstructuredClient(
         api_key_auth=FAKE_KEY,
     )
@@ -108,3 +111,78 @@ def test_suggest_defining_url_issues_a_warning_on_a_401():
             match="If intending to use the paid API, please define `server_url` in your request.",
         ):
             client.general.partition(req)
+
+
+@pytest.mark.parametrize("call_threads", [1, 2, 5])
+@pytest.mark.parametrize(
+    "filename, expected_ok",
+    [
+        ("_sample_docs/list-item-example-1.pdf", True),       # 1 page
+        ("_sample_docs/layout-parser-paper-fast.pdf", True),  # 2 pages
+        ("_sample_docs/layout-parser-paper.pdf", True),       # 16 pages
+        ("_sample_docs/fake.doc", True),
+        ("_sample_docs/fake.doc", False),  # This will append .pdf to filename to fool first line of filetype detection, to simulate decoding error
+    ],
+)
+def test_integration_split_pdf_has_same_output_as_non_split(
+    call_threads: int,
+    filename: str,
+    expected_ok: bool,
+    caplog
+):
+    """
+    Tests that output that we get from the split-by-page pdf is the same as from non-split.
+
+    Requires unstructured-api running in bg. See Makefile for how to run it.
+    Doesn't check for raw_response as there's no clear patter for how it changes with the number of pages / call_threads.
+    """
+    try:
+        response = requests.get("http://localhost:8000/general/docs")
+        assert response.status_code == 200, "The unstructured-api is not running on localhost:8000"
+    except requests.exceptions.ConnectionError:
+        assert False, "The unstructured-api is not running on localhost:8000"
+
+    client = UnstructuredClient(
+        api_key_auth=FAKE_KEY,
+        server_url="localhost:8000"
+    )
+
+    with open(filename, "rb") as f:
+        files = shared.Files(
+            content=f.read(),
+            file_name=filename,
+        )
+
+    if not expected_ok:
+        files.file_name += ".pdf"
+
+    req = shared.PartitionParameters(
+        files=files,
+        strategy='fast',
+        languages=["eng"],
+        split_pdf_page=True,
+    )
+
+    os.environ["UNSTRUCTURED_CLIENT_SPLIT_CALL_THREADS"] = str(call_threads)
+
+    try:
+        resp_split = client.general.partition(req)
+    except (HTTPValidationError, AttributeError) as exc:
+        if not expected_ok:
+            assert "error arose when splitting by pages" in caplog.text
+            assert "File does not appear to be a valid PDF" in str(exc)
+            return
+        else:
+            pytest.exit("unexpected error", returncode=1)
+
+    req.split_pdf_page = False
+    resp_single = client.general.partition(req)
+
+    assert len(resp_split.elements) == len(resp_single.elements)
+    assert resp_split.content_type == resp_single.content_type
+    assert resp_split.status_code == resp_single.status_code
+
+    # Difference in the parent_id is expected, because parent_ids are assigned when element crosses page boundary
+    diff = DeepDiff(t1=resp_split.elements, t2=resp_single.elements,
+                    exclude_regex_paths=r"root\[\d+\]\['metadata'\]\['parent_id'\]")
+    assert len(diff) == 0

_test_unstructured_client/test_utils_retries.py

@@ -12,11 +12,11 @@
 FAKE_KEY = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
 
 
-def test_retry_with_backoff_does_retry(caplog):
+def test_unit_retry_with_backoff_does_retry(caplog):
     caplog.set_level(logging.INFO)
     filename = "README.md"
     backoff_strategy = BackoffStrategy(
-        initial_interval=100, max_interval=1000, exponent=1.5, max_elapsed_time=3000
+        initial_interval=10, max_interval=100, exponent=1.5, max_elapsed_time=300
     )
     retries = RetryConfig(
         strategy="backoff", backoff=backoff_strategy, retry_connection_errors=True
@@ -41,11 +41,11 @@ def test_retry_with_backoff_does_retry(caplog):
         assert len(mock.request_history) > 1 
 
 
-def test_backoff_strategy_logs_retries(caplog):
+def test_unit_backoff_strategy_logs_retries(caplog):
     caplog.set_level(logging.INFO)
     filename = "README.md"
     backoff_strategy = BackoffStrategy(
-        initial_interval=100, max_interval=1000, exponent=1.5, max_elapsed_time=3000
+        initial_interval=10, max_interval=100, exponent=1.5, max_elapsed_time=300
     )
     retries = RetryConfig(
         strategy="backoff", backoff=backoff_strategy, retry_connection_errors=True

docs/models/shared/partitionparameters.md

@@ -21,5 +21,6 @@
 | `output_format`                                                                                                                                                                                                       | *Optional[str]*                                                                                                                                                                                                       | :heavy_minus_sign:                                                                                                                                                                                                    | The format of the response. Supported formats are application/json and text/csv. Default: application/json.                                                                                                           | application/json                                                                                                                                                                                                      |
 | `pdf_infer_table_structure`                                                                                                                                                                                           | *Optional[bool]*                                                                                                                                                                                                      | :heavy_minus_sign:                                                                                                                                                                                                    | If True and strategy=hi_res, any Table Elements extracted from a PDF will include an additional metadata field, 'text_as_html', where the value (string) is a just a transformation of the data into an HTML <table>. |                                                                                                                                                                                                                       |
 | `skip_infer_table_types`                                                                                                                                                                                              | List[*str*]                                                                                                                                                                                                           | :heavy_minus_sign:                                                                                                                                                                                                    | The document types that you want to skip table extraction with. Default: ['pdf', 'jpg', 'png']                                                                                                                        |                                                                                                                                                                                                                       |
+| `split_pdf_page`                                                                                                                                                                                                      | *Optional[bool]*                                                                                                                                                                                                      | :heavy_minus_sign:                                                                                                                                                                                                    | Should the pdf file be split at client. Ignored on backend.                                                                                                                                                           |                                                                                                                                                                                                                       |
 | `strategy`                                                                                                                                                                                                            | *Optional[str]*                                                                                                                                                                                                       | :heavy_minus_sign:                                                                                                                                                                                                    | The strategy to use for partitioning PDF/image. Options are fast, hi_res, auto. Default: auto                                                                                                                         | hi_res                                                                                                                                                                                                                |
 | `xml_keep_tags`                                                                                                                                                                                                       | *Optional[bool]*                                                                                                                                                                                                      | :heavy_minus_sign:                                                                                                                                                                                                    | If True, will retain the XML tags in the output. Otherwise it will simply extract the text from within the tags. Only applies to partition_xml.                                                                       |                                                                                                                                                                                                                       |