Add Data Loss Prevention Example

ci/vale/styles/config/vocabularies/morpheus/accept.txt

@@ -71,6 +71,7 @@ pytest
 [Ss]ubclassing
 [Ss]ubcard(s?)
 [Ss]ubgraph(s?)
+[Ss]ubnet(s?)
 [Ss]ubword(s?)
 [Ss]uperset(s?)
 [Tt]imestamp(s?)

conda/environments/all_cuda-128_arch-aarch64.yaml

@@ -39,6 +39,7 @@ dependencies:
 - cxx-compiler
 - cython=3.0
 - datacompy=0.13.1
+- datasets=3.6
 - dill=0.3.7
 - docker-py>=7.1,<8
 - doxygen=1.9.2
@@ -52,7 +53,7 @@ dependencies:
 - gtest=1.14
 - gxx=12.1
 - httpx>=0.23,<0.28
-- huggingface_hub=0.20.2
+- huggingface_hub>=0.24,<1.0
 - indicators=2.3
 - ipython
 - isort
@@ -79,6 +80,7 @@ dependencies:
 - numexpr
 - numpydoc=1.5
 - onnx=1.15
+- onnxruntime=1.22=*cuda
 - openai==1.13.*
 - papermill=2.6.0
 - pip
@@ -114,14 +116,15 @@ dependencies:
 - sqlalchemy<2.0
 - sysroot_linux-64>=2.28
 - tqdm=4
-- transformers=4.36.2
+- transformers>=4.40,<5.0
 - tritonclient=2.34
 - typing_utils=0.1
 - urllib3>=2.3,<3
 - versioneer
 - versioneer-518
 - watchdog=3.0
 - websockets
+- yaml=0.2
 - yapf=0.43
 - pip:
   - --extra-index-url https://download.pytorch.org/whl/cu124

conda/environments/all_cuda-128_arch-x86_64.yaml

@@ -39,6 +39,7 @@ dependencies:
 - cxx-compiler
 - cython=3.0
 - datacompy=0.13.1
+- datasets=3.6
 - dill=0.3.7
 - docker-py>=7.1,<8
 - doxygen=1.9.2
@@ -52,7 +53,7 @@ dependencies:
 - gtest=1.14
 - gxx=12.1
 - httpx>=0.23,<0.28
-- huggingface_hub=0.20.2
+- huggingface_hub>=0.24,<1.0
 - include-what-you-use=0.20
 - indicators=2.3
 - ipython
@@ -81,6 +82,7 @@ dependencies:
 - numexpr
 - numpydoc=1.5
 - onnx=1.15
+- onnxruntime=1.22=*cuda
 - openai==1.13.*
 - papermill=2.6.0
 - pip
@@ -117,7 +119,7 @@ dependencies:
 - sqlalchemy<2.0
 - sysroot_linux-64>=2.28
 - tqdm=4
-- transformers=4.36.2
+- transformers>=4.40,<5.0
 - tritonclient=2.34
 - typing_utils=0.1
 - urllib3>=2.3,<3
@@ -128,6 +130,7 @@ dependencies:
 - versioneer-518
 - watchdog=3.0
 - websockets
+- yaml=0.2
 - yapf=0.43
 - pip:
   - --extra-index-url https://download.pytorch.org/whl/cu124

conda/environments/dev_cuda-128_arch-aarch64.yaml

@@ -103,6 +103,7 @@ dependencies:
 - versioneer-518
 - watchdog=3.0
 - websockets
+- yaml=0.2
 - yapf=0.43
 - pip:
   - --extra-index-url https://download.pytorch.org/whl/cu124

conda/environments/dev_cuda-128_arch-x86_64.yaml

@@ -108,6 +108,7 @@ dependencies:
 - versioneer-518
 - watchdog=3.0
 - websockets
+- yaml=0.2
 - yapf=0.43
 - pip:
   - --extra-index-url https://download.pytorch.org/whl/cu124

conda/environments/examples_cuda-128_arch-aarch64.yaml

@@ -19,14 +19,15 @@ dependencies:
 - cuml=25.02.*
 - cupy<13.4
 - datacompy=0.13.1
+- datasets=3.6
 - dill=0.3.7
 - docker-py>=7.1,<8
 - elasticsearch==8.9.0
 - feedparser=6.0
 - grpcio
 - grpcio-status
 - httpx>=0.23,<0.28
-- huggingface_hub=0.20.2
+- huggingface_hub>=0.24,<1.0
 - jsonpatch>=1.33
 - kfp
 - libwebp=1.3.2
@@ -38,6 +39,7 @@ dependencies:
 - numexpr
 - numpydoc=1.5
 - onnx=1.15
+- onnxruntime=1.22=*cuda
 - openai==1.13.*
 - papermill=2.6.0
 - pip
@@ -57,12 +59,13 @@ dependencies:
 - scikit-learn=1.3.2
 - sqlalchemy<2.0
 - tqdm=4
-- transformers=4.36.2
+- transformers>=4.40,<5.0
 - tritonclient=2.34
 - typing_utils=0.1
 - urllib3>=2.3,<3
 - watchdog=3.0
 - websockets
+- yaml=0.2
 - pip:
   - --extra-index-url https://download.pytorch.org/whl/cu124
   - --find-links https://data.dgl.ai/wheels/torch-2.3/repo.html

conda/environments/examples_cuda-128_arch-x86_64.yaml

@@ -19,14 +19,15 @@ dependencies:
 - cuml=25.02.*
 - cupy<13.4
 - datacompy=0.13.1
+- datasets=3.6
 - dill=0.3.7
 - docker-py>=7.1,<8
 - elasticsearch==8.9.0
 - feedparser=6.0
 - grpcio
 - grpcio-status
 - httpx>=0.23,<0.28
-- huggingface_hub=0.20.2
+- huggingface_hub>=0.24,<1.0
 - jsonpatch>=1.33
 - kfp
 - libwebp=1.3.2
@@ -39,6 +40,7 @@ dependencies:
 - numexpr
 - numpydoc=1.5
 - onnx=1.15
+- onnxruntime=1.22=*cuda
 - openai==1.13.*
 - papermill=2.6.0
 - pip
@@ -59,12 +61,13 @@ dependencies:
 - scikit-learn=1.3.2
 - sqlalchemy<2.0
 - tqdm=4
-- transformers=4.36.2
+- transformers>=4.40,<5.0
 - tritonclient=2.34
 - typing_utils=0.1
 - urllib3>=2.3,<3
 - watchdog=3.0
 - websockets
+- yaml=0.2
 - pip:
   - --extra-index-url https://download.pytorch.org/whl/cu124
   - --find-links https://data.dgl.ai/wheels/torch-2.3/repo.html

conda/environments/model-utils_cuda-128_arch-aarch64.yaml

@@ -12,14 +12,14 @@ dependencies:
 - cuml=25.02.*
 - jupyterlab
 - matplotlib
-- onnx
+- onnx=1.15
 - pandas
 - pip
 - python=3.12
 - scikit-learn=1.3.2
 - seaborn
 - seqeval=1.2.2
-- transformers=4.36.2
+- transformers>=4.40,<5.0
 - xgboost
 - pip:
   - tensorrt-cu12

conda/environments/model-utils_cuda-128_arch-x86_64.yaml

@@ -12,14 +12,14 @@ dependencies:
 - cuml=25.02.*
 - jupyterlab
 - matplotlib
-- onnx
+- onnx=1.15
 - pandas
 - pip
 - python=3.12
 - scikit-learn=1.3.2
 - seaborn
 - seqeval=1.2.2
-- transformers=4.36.2
+- transformers>=4.40,<5.0
 - xgboost
 - pip:
   - tensorrt-cu12

conda/environments/runtime_cuda-128_arch-aarch64.yaml

@@ -48,6 +48,7 @@ dependencies:
 - urllib3>=2.3,<3
 - watchdog=3.0
 - websockets
+- yaml=0.2
 - pip:
   - --extra-index-url https://download.pytorch.org/whl/cu124
   - databricks-cli < 0.100

conda/environments/runtime_cuda-128_arch-x86_64.yaml

@@ -48,6 +48,7 @@ dependencies:
 - urllib3>=2.3,<3
 - watchdog=3.0
 - websockets
+- yaml=0.2
 - pip:
   - --extra-index-url https://download.pytorch.org/whl/cu124
   - databricks-cli < 0.100

dependencies.yaml

@@ -34,6 +34,7 @@ files:
       - development
       - docs
       - example-dfp-prod
+      - example-dlp
       - example-gnn
       - example-llms
       - python
@@ -152,6 +153,7 @@ files:
       - cve-mitigation
       - example-abp-nvsmi
       - example-dfp-prod
+      - example-dlp
       - example-gnn
       - example-llms
       - python
@@ -418,6 +420,7 @@ dependencies:
           - urllib3>=2.3,<3
           - watchdog=3.0
           - websockets
+          - yaml=0.2
           - pip
           - pip:
             - &torch-extra-index --extra-index-url https://download.pytorch.org/whl/cu124
@@ -559,14 +562,14 @@ dependencies:
     common:
       - output_types: [conda]
         packages:
-          - &transformers transformers=4.36.2 # newer versions are incompatible with our pinned version of huggingface_hub
+          - &transformers transformers>=4.40,<5.0
           - anyio>=3.7
           - arxiv=1.4
           - httpx>=0.23,<0.28 # work-around for https://github.com/openai/openai-python/issues/1915
-          - huggingface_hub=0.20.2 # work-around for https://github.com/UKPLab/sentence-transformers/issues/1762
+          - huggingface_hub>=0.24,<1.0
           - jsonpatch>=1.33
           - numexpr
-          - onnx=1.15
+          - &onnx onnx=1.15
           - *openai
           - pypdf=3.17.4
           - *python-docx
@@ -606,7 +609,7 @@ dependencies:
           - scikit-learn=1.3.2
           - jupyterlab
           - matplotlib
-          - onnx
+          - *onnx
           - pandas
           - seaborn
           - seqeval=1.2.2
@@ -620,3 +623,11 @@ dependencies:
       - output_types: [conda]
         packages:
           - libwebp=1.3.2 # Required for CVE mitigation: https://nvd.nist.gov/vuln/detail/CVE-2023-4863
+
+  example-dlp:
+    common:
+      - output_types: [conda]
+        packages:
+          - datasets=3.6
+          - onnxruntime=1.22=*cuda
+          # gliner intentionally omitted due to https://github.com/urchade/GLiNER/issues/267

examples/CMakeLists.txt

@@ -17,5 +17,6 @@ list(APPEND CMAKE_MESSAGE_CONTEXT "examples")
 list(PREPEND CMAKE_PREFIX_PATH "${PROJECT_BINARY_DIR}")
 
 add_subdirectory(developer_guide)
+add_subdirectory(data_loss_prevention)
 
 list(POP_BACK CMAKE_MESSAGE_CONTEXT)

examples/README.md

@@ -34,6 +34,7 @@ limitations under the License.
 ## Environments
 Morpheus supports multiple environments, each environment is intended to support a given use-case. Each example documents which environments it is able to run in. With the exception of the Morpheus Release Container, the examples require fetching both the `datasets` and `examples` dataset via the `fetch_data.sh` script:
 ```bash
+git lfs install
 ./scripts/fetch_data.py fetch examples datasets
 ```
 

examples/data_loss_prevention/CMakeLists.txt

@@ -0,0 +1,79 @@
+# SPDX-FileCopyrightText: Copyright (c) 2022-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+cmake_minimum_required(VERSION 3.25 FATAL_ERROR)
+
+list(APPEND CMAKE_MESSAGE_CONTEXT "dlp")
+
+# Set the cache to be the same to allow for CCache to be used effectively
+set(MORPHEUS_CACHE_DIR "${CMAKE_SOURCE_DIR}/.cache" CACHE PATH "Directory to contain all CPM and CCache data")
+mark_as_advanced(MORPHEUS_CACHE_DIR)
+
+# Add the Conda environment to the prefix path and add the CMake files
+list(PREPEND CMAKE_PREFIX_PATH "$ENV{CONDA_PREFIX}")
+
+project(dlp
+  VERSION 25.06.00
+  LANGUAGES C CXX
+)
+
+set(CMAKE_CXX_STANDARD 20)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+set(CMAKE_CXX_EXTENSIONS ON)
+set(CMAKE_POSITION_INDEPENDENT_CODE TRUE)
+set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE)
+set(CMAKE_INSTALL_RPATH "$ORIGIN")
+
+# Set the option prefix to match the outer project before including. Must be before find_package(morpheus)
+set(OPTION_PREFIX "MORPHEUS")
+
+# Set the policy to allow for CMP0144, avoids warning about MORPHEUS_ROOT being set
+cmake_policy(SET CMP0144 NEW)
+
+find_package(morpheus REQUIRED)
+find_package(glog REQUIRED) # work-around for #2149
+
+morpheus_utils_initialize_cpm(MORPHEUS_CACHE_DIR)
+
+# Ensure CPM is initialized
+rapids_cpm_init()
+
+morpheus_utils_python_configure()
+
+rapids_find_package(CUDAToolkit REQUIRED)
+rapids_find_package(cudf REQUIRED)
+
+set(CMAKE_POSITION_INDEPENDENT_CODE TRUE)
+set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
+
+morpheus_utils_create_python_package(dlp_stages
+  PROJECT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}"
+  SOURCE_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/dlp_stages"
+)
+
+add_subdirectory(dlp_stages/_lib)
+
+# Complete the python package
+if(MORPHEUS_PYTHON_INPLACE_BUILD)
+  list(APPEND extra_args "IS_INPLACE")
+endif()
+
+if(TARGET morpheus-package-install)
+  list(APPEND extra_args "PYTHON_DEPENDENCIES" "morpheus-package-install")
+endif()
+
+morpheus_utils_build_python_package(dlp_stages ${extra_args})
+
+list(POP_BACK CMAKE_MESSAGE_CONTEXT)