Feature / Full support for STRUCT data type (#591)

* Text file producers

* Test file consumers

* Text file reader / writer

* Arrow VSR and schema mapping

* Common text codec

* Move JSON codec to common text codec base

* Move CSV codec to common text codec base

* Update common codec API

* Make arrow codecs use the new base API

* Java test updates

* Use trac schemas in data storage framework instead of arrow

* Schema mapping fixes

* Schema validator

* Add a test case for schema mapping

* Test data for JSON structs

* Proto updates

* Runtime struct updates

* Runtime framework updates

* Test case fixes

* Test case fixes

* Handle dict encoded fields in data comparison test util

* Record named enums in define_struct

* Allow qualified identifiers for named types in schema validator

* Throw a more meaningful error for unrecognized fields in json struct parser

* Fix sample data for structured run config

* Fix test schema for runtime structs

* Support aliases for format types in struct processor

* Special case the storage layout for struct objects in the runtime

* Handle storage layout for structs in the data service

* Add an end-to-end test for a struct model

* Add a test case for chained structs in the examples

* Handle different cases for alt struct formats

* In context impl, allow that structs may already be decoded

* Add a test case for running struct models through a flow

* Remove debug logging

* Update job consistency validator to handle nested types

* Bump versions for compliance

* Remove previous version forcing for Azure

* Force version on Nimbus JOSE in Azure plugin

Author: martin-traverse

examples/models/python/config/chaining_struct.yaml

@@ -0,0 +1,19 @@
+
+job:
+  runFlow:
+
+    flow: ./chaining_struct_flow.yaml
+
+    parameters:
+      t0_date: "2025-07-01"
+      projection_period: 365
+
+    inputs:
+      run_config: "inputs/structured_run_config.yaml"
+
+    outputs:
+      modified_config: "outputs/chaining/modified_run_config.yaml"
+
+    models:
+      model_1: tutorial.structured_objects.StructModel
+      model_2: tutorial.structured_objects.StructModel

examples/models/python/config/chaining_struct_flow.yaml

@@ -0,0 +1,29 @@
+
+nodes:
+
+  run_config:
+    nodeType: "INPUT_NODE"
+
+  model_1:
+    nodeType: "MODEL_NODE"
+    inputs: [run_config]
+    outputs: [modified_config]
+
+  model_2:
+    nodeType: "MODEL_NODE"
+    inputs: [run_config]
+    outputs: [modified_config]
+
+  modified_config:
+    nodeType: "OUTPUT_NODE"
+
+edges:
+
+  - source: { node: run_config }
+    target: { node: model_1, socket: run_config }
+
+  - source: { node: model_1, socket: modified_config }
+    target: { node: model_2, socket: run_config }
+
+  - source: { node: model_2, socket: modified_config }
+    target: { node: modified_config }

examples/models/python/data/inputs/structured_run_config.json

@@ -4,26 +4,20 @@
    "base_scenario": {
       "scenario_name": "base",
       "default_weight": 1.0,
-      "evolution_model": "perturb",
-      "apply_smoothing": true,
-      "correlated_scenario": {
-         "scenario_name": "correlated",
-         "default_weight": 1.0,
-         "evolution_model": "perturb_smooth",
-         "apply_smoothing": false
-      }
+      "evolution_model": "PERTURB",
+      "apply_smoothing": true
    },
    "stress_scenarios": {
       "downturn": {
          "scenario_name": "downturn",
          "default_weight": 0.37,
-         "evolution_model": "perturb",
+         "evolution_model": "PERTURB",
          "apply_smoothing": false
       },
       "high_interest": {
          "scenario_name": "high_interest",
          "default_weight": 0.66,
-         "evolution_model": "perturb",
+         "evolution_model": "PERTURB",
          "apply_smoothing": false
       }
    }

examples/models/python/data/inputs/structured_run_config.yaml

@@ -4,11 +4,6 @@ base_scenario:
   default_weight: 1.0
   evolution_model: "perturb"
   apply_smoothing: true
-  correlated_scenario:
-    scenario_name: "correlated"
-    default_weight: 1.0
-    evolution_model: "perturb_smooth"
-    apply_smoothing: false
 
 stress_scenarios:
   downturn:

gradle/versions.gradle

@@ -63,7 +63,7 @@ ext {
 
     aws_sdk_version = '2.31.59'
     gcp_sdk_version = '26.61.0'
-    azure_sdk_version = '1.2.35'
+    azure_sdk_version = '1.2.36'
 
     apache_sshd_version = "2.12.1"
 

tracdap-api/tracdap-metadata/src/main/proto/tracdap/metadata/data.proto

@@ -34,13 +34,16 @@ import "tracdap/metadata/object_id.proto";
  */
 enum SchemaType {
 
+    option allow_alias = true;
+
     SCHEMA_TYPE_NOT_SET = 0;
 
     /// Tabular data
     TABLE = 1;
+    TABLE_SCHEMA = 1;
 
     /// Structured objects
-    STRUCT = 2;
+    STRUCT_SCHEMA = 2;
 }
 
 /**
@@ -63,8 +66,18 @@ message FieldSchema {
     optional bool notNull = 8;
 
     optional string formatCode = 7;
+
+    optional Value defaultValue = 9;
+    optional string namedType = 10;
+    optional string namedEnum = 11;
+
+    repeated FieldSchema children = 12;
 }
 
+message EnumValues {
+
+    repeated string values = 1;
+}
 
 /**
  * Schema for a tabular dataset
@@ -75,38 +88,6 @@ message TableSchema {
 }
 
 
-/**
- * Schema for an individual field in a structured object dataset
- */
-message StructField {
-
-    TypeDescriptor fieldType = 3;
-
-    string label = 4;
-
-    bool businessKey = 5;
-    bool categorical = 6;
-
-    // This could become mandatory with the next metadata update
-    optional bool notNull = 8;
-
-    optional string formatCode = 7;
-
-    Value defaultValue = 9;
-
-    optional string structType = 10;
-}
-
-
-/**
-  * Schema for a structured object dataset
-  */
-message StructSchema {
-
-    map<string, StructField> fields = 1;
-
-    map<string, StructSchema> namedTypes = 2;
-}
 
 
 /**
@@ -129,8 +110,13 @@ message SchemaDefinition {
     oneof schemaDetails {
 
         TableSchema table = 3;
-        StructSchema struct = 4;
     }
+
+    repeated FieldSchema fields = 5;
+
+    map<string, SchemaDefinition> namedTypes = 6;
+
+    map<string, EnumValues> namedEnums = 7;
 }
 
 
@@ -139,8 +125,11 @@ message SchemaDefinition {
  */
 enum PartType {
 
+    option allow_alias = true;
+
     /// Dataset has a single partition called the root partition (this is the default)
     PART_ROOT = 0;
+    NOT_PARTITIONED = 0;
 
     /// Partition by range over an ordered variable (not available yet)
     PART_BY_RANGE = 1;

tracdap-api/tracdap-metadata/src/main/proto/tracdap/metadata/type.proto

@@ -103,15 +103,16 @@ enum BasicType {
     /**
      * An key-value map with string keys, values may be primitive or composite values.
      *
-     * Maps may be uniform, in which case all the values are of the same type, or non-
-     * uniform in which case values can be of any type. For uniform maps the type
-     * descriptor will specify the type contained in the map. For non-uniform maps the
-     * type descriptor can only specify that the map is non-uniform, values must be
-     * examined at run time to determine their type.
+     * All values in a map must have the same type (i.e. the same type descriptor).
      *
      * @see TypeDescriptor
      */
     MAP = 9;
+
+    /**
+     * A nested structure with a named set of fields, which may be primitive or composite values.
+     */
+    STRUCT = 10;
 }
 
 
@@ -140,6 +141,13 @@ message TypeDescriptor {
      * be inspected individually to determine their type.
      */
     optional TypeDescriptor mapType = 3;
+
+    reserved 4;
+    reserved "mapKeyType";
+
+    map<string, TypeDescriptor> structTypes = 5;
+
+    optional string typeName = 6;
 }
 
 

tracdap-libs/tracdap-lib-common/src/main/java/org/finos/tracdap/common/metadata/MetadataConstants.java

@@ -58,9 +58,11 @@ public class MetadataConstants {
     // ^...$ would allow matches like "my_var\n_gotcha"
     public static final Pattern VALID_IDENTIFIER = Pattern.compile("\\A[a-zA-Z_]\\w*\\Z");
 
+    public static final Pattern QUALIFIED_IDENTIFIER = Pattern.compile("\\A[a-zA-Z_]\\w*(\\.[a-zA-Z_]\\w*)*\\Z");
+
     // Identifiers starting trac_ are reserved for use by the TRAC platform
     // Identifiers starting _ are also reserved by convention, for private / protected / system variables
-    public static final Pattern TRAC_RESERVED_IDENTIFIER = Pattern.compile("\\A(trac[_\\-.]|[_\\-.]).*", Pattern.CASE_INSENSITIVE);
+    public static final Pattern TRAC_RESERVED_IDENTIFIER = Pattern.compile("\\A(trac(dap)?[_\\-.]|[_\\-.]).*", Pattern.CASE_INSENSITIVE);
 
     public static final String TRAC_CREATE_TIME = "trac_create_time";
     public static final String TRAC_CREATE_USER_ID = "trac_create_user_id";

tracdap-libs/tracdap-lib-data/src/main/java/org/finos/tracdap/common/codec/CodecManager.java

@@ -23,25 +23,39 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
 
 public class CodecManager implements ICodecManager {
 
     private final Logger log = LoggerFactory.getLogger(getClass());
 
-    private final IPluginManager plugins;
-    private final ConfigManager configManager;
+    private final Map<String, ICodec> codecs;
 
     public CodecManager(IPluginManager plugins, ConfigManager configManager) {
-        this.plugins = plugins;
-        this.configManager = configManager;
+
+        var protocols = plugins.availableProtocols(ICodec.class);
+        var codecs = new HashMap<String, ICodec>();
+
+        for (var protocol : protocols) {
+            var codec = plugins.createService(ICodec.class, protocol, configManager);
+            codecs.put(protocol.toLowerCase(), codec);
+        }
+
+        this.codecs = Collections.unmodifiableMap(codecs);
     }
 
     @Override
     public ICodec getCodec(String format) {
 
-        if (plugins.isServiceAvailable(ICodec.class, format))
-            return plugins.createService(ICodec.class, format, configManager);
+        var protocol = format.toLowerCase();
+
+        if (codecs.containsKey(protocol)) {
 
+            return codecs.get(protocol);
+        }
         else {
 
             // Make a slightly prettier message that the regular plugin not available message

tracdap-libs/tracdap-lib-data/src/main/java/org/finos/tracdap/common/codec/ICodec.java

@@ -18,6 +18,7 @@
 package org.finos.tracdap.common.codec;
 
 import org.finos.tracdap.common.data.ArrowVsrSchema;
+import org.finos.tracdap.metadata.SchemaDefinition;
 import org.finos.tracdap.common.data.DataPipeline;
 
 import org.apache.arrow.memory.BufferAllocator;
@@ -47,6 +48,15 @@ Encoder<?> getEncoder(
 
     Decoder<?> getDecoder(
             BufferAllocator allocator,
-            ArrowVsrSchema schema,
+            Map<String, String> options);
+
+    Decoder<?> getDecoder(
+            SchemaDefinition tracSchema,
+            BufferAllocator allocator,
+            Map<String, String> options);
+
+    Decoder<?> getDecoder(
+            ArrowVsrSchema arrowSchema,
+            BufferAllocator allocator,
             Map<String, String> options);
 }