second commit

Author: zephyrie

SERVER_SETUP.md

@@ -59,6 +59,7 @@ The server requires these critical packages:
 - `transformers >= 4.50.0`
 - `itk >= 5.3.0`
 - `monai >= 1.5.1`
+- `scikit-image >= 0.25.0` - **Required for Clara Generate (MAISI)**: Image processing for synthetic CT generation
 
 ### 3. Socket.IO Version Compatibility
 
@@ -149,9 +150,11 @@ Create or edit `server/.env`:
 # Enable/disable features
 ENABLE_VISTA3D=true
 ENABLE_LLM_CHAT=true
+ENABLE_CLARA_GENERATE=true
 
-# Custom bundle directory (optional)
+# Custom bundle directories (optional)
 VISTA3D_BUNDLE_DIR=bundles/
+CLARA_GENERATE_BUNDLE_DIR=bundles/
 ```
 
 #### Configuration via Environment Variables
@@ -180,11 +183,14 @@ ENABLE_VISTA3D=false uv run python -m volview_server -H 0.0.0.0 -P 4014 2025_nvi
 **Available Feature Flags:**
 - `ENABLE_VISTA3D=true/false` - Enable/disable Vista3D segmentation (default: true)
 - `ENABLE_LLM_CHAT=true/false` - Enable/disable LLM Chat features (default: true)
-- `VISTA3D_BUNDLE_DIR=path` - Custom MONAI bundle directory (default: "bundles/")
+- `ENABLE_CLARA_GENERATE=true/false` - Enable/disable Clara Generate synthetic CT (default: true)
+- `VISTA3D_BUNDLE_DIR=path` - Custom MONAI Vista3D bundle directory (default: "bundles/")
+- `CLARA_GENERATE_BUNDLE_DIR=path` - Custom MONAI MAISI bundle directory (default: "bundles/")
 
 **Available RPC Endpoints:**
-- ✅ `segmentWithMONAI` - MONAI Vista3D segmentation (when ENABLE_VISTA3D=true)
+- ✅ `segmentWithMONAI` - MONAI Vista3D whole-body CT segmentation (when ENABLE_VISTA3D=true)
 - ✅ `multimodalLlmAnalysis` - Multimodal LLM analysis with model dispatch (when ENABLE_LLM_CHAT=true)
+- ✅ `generateWithClara` - NVIDIA MAISI synthetic CT generation with paired masks (when ENABLE_CLARA_GENERATE=true)
 
 **Lazy Loading:** Heavy dependencies (torch, transformers) are only imported on first LLM inference call, allowing fast server startup.
 
@@ -367,9 +373,10 @@ If running out of GPU memory:
 
 1. **`volview_server/__main__.py`** - Entry point, starts aiohttp server with Socket.IO
 2. **`volview_server/rpc_server.py`** - Handles RPC calls and Socket.IO events
-3. **`2025_nvidiagtcdc/api.py`** - Unified API exposing both `segmentWithMONAI` and `multimodalLlmAnalysis` with modular loading
-4. **`2025_nvidiagtcdc/volview_insight_clara_nv_reason_cxr_3b_inference.py`** - Clara model inference logic (lazy loaded)
+3. **`2025_nvidiagtcdc/api.py`** - Unified API exposing `segmentWithMONAI`, `multimodalLlmAnalysis`, and `generateWithClara` with modular loading
+4. **`2025_nvidiagtcdc/volview_insight_clara_nv_reason_cxr_3b_inference.py`** - Clara Reason model inference logic (lazy loaded)
 5. **`2025_nvidiagtcdc/volview_insight_medgemma_inference.py`** - MedGemma inference logic (if implemented)
+6. **`2025_nvidiagtcdc/volview_clara_generate_inference.py`** - Clara Generate (MAISI) synthetic CT generation logic
 
 ### Frontend Components
 

server/2025_nvidiagtcdc/api.py

@@ -1,13 +1,15 @@
 """
 Unified VolView API for GTC 2025 Demo.
 
-Combines Vista3D segmentation and multimodal LLM chat features with modular,
-lazy loading for optimal performance.
+Combines Vista3D segmentation, multimodal LLM chat, and Clara Generate
+(synthetic CT generation) features with modular, lazy loading for optimal performance.
 
 Features can be controlled via .env file or environment variables:
 - ENABLE_VISTA3D=true (default: true)
 - ENABLE_LLM_CHAT=true (default: true)
+- ENABLE_CLARA_GENERATE=true (default: true)
 - VISTA3D_BUNDLE_DIR=path (default: "bundles/")
+- CLARA_GENERATE_BUNDLE_DIR=path (default: "bundles/")
 
 Configuration Priority:
 1. Environment variables (highest)
@@ -42,12 +44,14 @@
 # Feature flags (read from environment, default to true)
 ENABLE_VISTA3D = os.getenv("ENABLE_VISTA3D", "true").lower() in ("true", "1", "yes")
 ENABLE_LLM_CHAT = os.getenv("ENABLE_LLM_CHAT", "true").lower() in ("true", "1", "yes")
+ENABLE_CLARA_GENERATE = os.getenv("ENABLE_CLARA_GENERATE", "true").lower() in ("true", "1", "yes")
 
 print("=" * 80)
 print("VolView Server - GTC 2025 Demo")
 print("=" * 80)
 print(f"ENABLE_VISTA3D: {ENABLE_VISTA3D}")
 print(f"ENABLE_LLM_CHAT: {ENABLE_LLM_CHAT}")
+print(f"ENABLE_CLARA_GENERATE: {ENABLE_CLARA_GENERATE}")
 print("=" * 80)
 
 # ============================================================================
@@ -390,6 +394,104 @@ async def multimodal_llm_analysis(
 else:
     print("✗ LLM Chat feature disabled")
 
+# ============================================================================
+# CLARA GENERATE FEATURE - Synthetic CT Image Generation
+# ============================================================================
+
+if ENABLE_CLARA_GENERATE:
+    print("Loading Clara Generate feature...")
+
+    import asyncio
+    from concurrent.futures import ProcessPoolExecutor
+    from typing import Tuple
+
+    # Import the inference function
+    from volview_clara_generate_inference import run_clara_generate_inference
+
+    # Process pool for CPU-intensive generation (separate from Vista3D)
+    clara_generate_process_pool = ProcessPoolExecutor(max_workers=1)
+
+    # Bundle configuration
+    CLARA_GENERATE_BUNDLE_DIR = os.getenv("CLARA_GENERATE_BUNDLE_DIR", "bundles/")
+
+    def do_clara_generate_inference(
+        output_size: list,
+        spacing: list,
+    ) -> Tuple[bytes, bytes]:
+        """
+        Runs Clara Generate (MAISI) inference in a separate process.
+
+        Returns:
+            Tuple of (image_blob, mask_blob) as bytes.
+        """
+        return run_clara_generate_inference(
+            output_size=output_size,
+            spacing=spacing,
+            bundle_dir=CLARA_GENERATE_BUNDLE_DIR,
+        )
+
+    async def run_clara_generate_process(
+        output_size: list, spacing: list
+    ) -> Tuple[bytes, bytes]:
+        """
+        Asynchronously runs Clara Generate inference in the process pool.
+        """
+        loop = asyncio.get_event_loop()
+        result = await loop.run_in_executor(
+            clara_generate_process_pool,
+            do_clara_generate_inference,
+            output_size,
+            spacing,
+        )
+        return result
+
+    @volview.expose("generateWithClara")
+    async def generate_with_clara(
+        output_size: list = None,
+        spacing: list = None,
+    ):
+        """
+        Exposes Clara Generate (MAISI) synthetic CT generation to the VolView client.
+
+        Args:
+            output_size: Volume size [x, y, z] (default: [256, 256, 128]).
+                        X,Y must be from [256, 384, 512].
+                        Z must be from [128, 256, 384, 512, 640, 768].
+            spacing: Voxel size [x, y, z] in mm (default: [1.5, 1.5, 1.5]).
+
+        Returns:
+            dict: Contains 'image' and 'mask' as byte arrays (for JSON serialization).
+        """
+        # Set defaults
+        if output_size is None:
+            output_size = [256, 256, 128]
+        if spacing is None:
+            spacing = [1.5, 1.5, 1.5]
+
+        print(
+            f"Received Clara Generate request: size={output_size}, spacing={spacing}"
+        )
+
+        image_blob, mask_blob = await run_clara_generate_process(
+            output_size, spacing
+        )
+
+        print(
+            f"Successfully generated synthetic CT. "
+            f"Returning {len(image_blob)} byte image and {len(mask_blob)} byte mask to client."
+        )
+
+        # Convert bytes to lists for JSON serialization over Socket.IO
+        return {
+            "image": list(image_blob),
+            "mask": list(mask_blob),
+        }
+
+    print("✓ Clara Generate feature loaded (generateWithClara endpoint available)")
+
+else:
+    print("✗ Clara Generate feature disabled")
+
 # ============================================================================
 # Summary
 # ============================================================================
@@ -400,10 +502,12 @@ async def multimodal_llm_analysis(
     available_endpoints.append("segmentWithMONAI")
 if ENABLE_LLM_CHAT:
     available_endpoints.append("multimodalLlmAnalysis")
+if ENABLE_CLARA_GENERATE:
+    available_endpoints.append("generateWithClara")
 
 if available_endpoints:
     print(f"Available RPC endpoints: {', '.join(available_endpoints)}")
 else:
-    print("WARNING: No features enabled! Set ENABLE_VISTA3D=true or ENABLE_LLM_CHAT=true")
+    print("WARNING: No features enabled! Set ENABLE_VISTA3D=true, ENABLE_LLM_CHAT=true, or ENABLE_CLARA_GENERATE=true")
 
 print("=" * 80)