feat(core): Handle split keys on tdf3 encrypt and decrypt

.github/workflows/checks.yaml

@@ -190,6 +190,60 @@ jobs:
           fi
         working-directory: cmdline
 
+      - uses: JarvusInnovations/background-action@2428e7b970a846423095c79d43f759abf979a635
+        name: start another KAS server in background
+        with:
+          run: >
+            <opentdf.yaml >opentdf-beta.yaml yq e '
+              (.server.port = 8282)
+              | (.mode = ["kas"])
+              | (.sdk_config = {"endpoint":"http://localhost:8080","plaintext":true,"client_id":"opentdf","client_secret":"secret"})
+            '
+            && go run ./service --config-file ./opentdf-beta.yaml start
+          wait-on: |
+            tcp:localhost:8282
+          log-output-if: true
+          wait-for: 90s
+          working-directory: platform
+      - name: Make sure that the second platform is up
+        run: |
+          grpcurl -plaintext localhost:8282 kas.AccessService/PublicKey
+      - name: Validate multikas through the command line interface
+        run: |
+          printf 'here is some data to encrypt' > data
+
+          java -jar target/cmdline.jar \
+            --client-id=opentdf-sdk \
+            --client-secret=secret \
+            --platform-endpoint=localhost:8080 \
+            -i \
+            encrypt --kas-url=localhost:8080,localhost:8282 -f data -m 'here is some metadata' > test.tdf
+
+          java -jar target/cmdline.jar \
+            --client-id=opentdf-sdk \
+            --client-secret=secret \
+            --platform-endpoint=localhost:8080 \
+            -i \
+            decrypt -f test.tdf > decrypted
+
+          java -jar target/cmdline.jar \
+            --client-id=opentdf-sdk \
+            --client-secret=secret \
+            --platform-endpoint=localhost:8080 \
+            -i \
+            metadata -f test.tdf > metadata
+
+          if ! diff -q data decrypted; then
+            printf 'decrypted data is incorrect [%s]' "$(< decrypted)"
+            exit 1
+          fi
+
+          if [ "$(< metadata)" != 'here is some metadata' ]; then
+            printf 'metadata is incorrect [%s]\n' "$(< metadata)"
+            exit 1
+          fi
+        working-directory: cmdline
+
   platform-xtest:
     permissions:
       contents: read

cmdline/src/main/java/io/opentdf/platform/Command.java

@@ -3,6 +3,7 @@
 import com.nimbusds.jose.JOSEException;
 import io.opentdf.platform.sdk.*;
 import io.opentdf.platform.sdk.TDF;
+
 import org.apache.commons.codec.DecoderException;
 import picocli.CommandLine;
 import picocli.CommandLine.Option;
@@ -49,7 +50,7 @@ class Command {
     @CommandLine.Command(name = "encrypt")
     void encrypt(
             @Option(names = {"-f", "--file"}, defaultValue = Option.NULL_VALUE) Optional<File> file,
-            @Option(names = {"-k", "--kas-url"}, required = true) List<String> kas,
+            @Option(names = {"-k", "--kas-url"}, required = true, split = ",") List<String> kas,
             @Option(names = {"-m", "--metadata"}, defaultValue = Option.NULL_VALUE) Optional<String> metadata) throws
             IOException, JOSEException {
 

sdk/src/main/java/io/opentdf/platform/sdk/Config.java

@@ -53,6 +53,16 @@ public AssertionConfig() {
         }
     }
 
+    public static class SplitStep {
+        public String kas;
+        public String splitID;
+
+        public SplitStep(String kas, String sid) {
+            this.kas = kas;
+            this.splitID = sid;
+          }
+    }
+
     public static class TDFConfig {
         public int defaultSegmentSize;
         public boolean enableEncryption;
@@ -67,6 +77,7 @@ public static class TDFConfig {
         public List<Assertion> assertionList;
         public AssertionConfig assertionConfig;
         public String mimeType;
+        public List<SplitStep> splitPlan;
 
         public TDFConfig() {
             this.defaultSegmentSize = DEFAULT_SEGMENT_SIZE;
@@ -78,6 +89,7 @@ public TDFConfig() {
             this.kasInfoList = new ArrayList<>();
             this.assertionList = new ArrayList<>();
             this.mimeType = DEFAULT_MIME_TYPE;
+            this.splitPlan = new ArrayList<>();
         }
     }
 

sdk/src/main/java/io/opentdf/platform/sdk/Manifest.java

@@ -134,6 +134,7 @@ static public class KeyAccess {
 
         public String encryptedMetadata;
         public String kid;
+        public String sid;
 
         @Override
         public boolean equals(Object o) {

sdk/src/main/java/io/opentdf/platform/sdk/TDF.java

@@ -13,6 +13,7 @@
 import com.nimbusds.jose.crypto.RSASSAVerifier;
 import com.nimbusds.jwt.JWTClaimsSet;
 import com.nimbusds.jwt.SignedJWT;
+
 import com.nimbusds.jose.crypto.RSASSASigner;
 import com.nimbusds.jose.crypto.MACSigner;
 import org.apache.commons.codec.DecoderException;
@@ -68,6 +69,12 @@ public TDF() {
 
     private static final Gson gson = new Gson();
 
+    public class SplitKeyException extends IOException { 
+        public SplitKeyException(String errorMessage) {
+            super(errorMessage);
+        }
+    }
+
     public static class DataSizeNotSupported extends RuntimeException {
         public DataSizeNotSupported(String errorMessage) {
             super(errorMessage);
@@ -164,57 +171,111 @@ PolicyObject createPolicyObject(List<String> attributes) {
         }
 
         private static final Base64.Encoder encoder = Base64.getEncoder();
-        private void prepareManifest(Config.TDFConfig tdfConfig) {
+        private void prepareManifest(Config.TDFConfig tdfConfig, SDK.KAS kas) {
             manifest.encryptionInformation.keyAccessType = kSplitKeyType;
             manifest.encryptionInformation.keyAccessObj =  new ArrayList<>();
 
             PolicyObject policyObject = createPolicyObject(tdfConfig.attributes);
             String base64PolicyObject  = encoder.encodeToString(gson.toJson(policyObject).getBytes(StandardCharsets.UTF_8));
             List<byte[]> symKeys = new ArrayList<>();
+            Map<String, Config.KASInfo> latestKASInfo = new HashMap<>();
+            if (tdfConfig.splitPlan.isEmpty()) {
+                // Default split plan: Split keys across all KASes
+                List<Config.SplitStep> splitPlan = new ArrayList<>(tdfConfig.kasInfoList.size());
+                for (Config.KASInfo kasInfo : tdfConfig.kasInfoList) {
+                    Config.SplitStep step = new Config.SplitStep(kasInfo.URL, "");
+                    if (tdfConfig.kasInfoList.size() > 1) {
+                        step.splitID = String.format("s-%d", splitPlan.size());
+                    }
+                    splitPlan.add(step);
+                    if (kasInfo.PublicKey != null && !kasInfo.PublicKey.isEmpty()) {
+                        latestKASInfo.put(kasInfo.URL, kasInfo);
+                    }
+                }
+                tdfConfig.splitPlan = splitPlan;
+            }
 
+            // Seed anything passed in manually
             for (Config.KASInfo kasInfo: tdfConfig.kasInfoList) {
-                if (kasInfo.PublicKey == null || kasInfo.PublicKey.isEmpty()) {
-                    throw new KasPublicKeyMissing("Kas public key is missing in kas information list");
+                if (kasInfo.PublicKey != null && !kasInfo.PublicKey.isEmpty()) {
+                    latestKASInfo.put(kasInfo.URL, kasInfo);
+                }
+            }
+
+            // split plan: restructure by conjunctions
+            Map<String, List<Config.KASInfo>> conjunction = new HashMap<>();
+            List<String> splitIDs = new ArrayList<>();
+
+            for (Config.SplitStep splitInfo : tdfConfig.splitPlan) {
+                // Public key was passed in with kasInfoList
+                // TODO First look up in attribute information / add to split plan?
+                Config.KASInfo ki = latestKASInfo.get(splitInfo.kas);
+                if (ki == null || ki.PublicKey == null || ki.PublicKey.isBlank()) {
+                    logger.info("no public key provided for KAS at {}, retrieving", splitInfo.kas);
+                    var getKI = new Config.KASInfo();
+                    getKI.URL = splitInfo.kas;
+                    getKI.PublicKey = kas.getPublicKey(getKI);
+                    getKI.KID = kas.getKid(getKI);
+                    latestKASInfo.put(splitInfo.kas, getKI);
+                    ki = getKI;
                 }
+                if (conjunction.containsKey(splitInfo.splitID)) {
+                    conjunction.get(splitInfo.splitID).add(ki);
+                } else {
+                    List<Config.KASInfo> newList = new ArrayList<>();
+                    newList.add(ki);
+                    conjunction.put(splitInfo.splitID, newList);
+                    splitIDs.add(splitInfo.splitID);
+                }
+            }
 
+            for (String splitID: splitIDs) {
                 // Symmetric key
                 byte[] symKey = new byte[GCM_KEY_SIZE];
                 sRandom.nextBytes(symKey);
-
-                Manifest.KeyAccess keyAccess = new Manifest.KeyAccess();
-                keyAccess.keyType = kWrapped;
-                keyAccess.url = kasInfo.URL;
-                keyAccess.kid = kasInfo.KID;
-                keyAccess.protocol = kKasProtocol;
+                symKeys.add(symKey);
 
                 // Add policyBinding
                 var hexBinding = Hex.encodeHexString(CryptoUtils.CalculateSHA256Hmac(symKey, base64PolicyObject.getBytes(StandardCharsets.UTF_8)));
                 var policyBinding = new Manifest.PolicyBinding();
                 policyBinding.alg = kHmacIntegrityAlgorithm;
                 policyBinding.hash = encoder.encodeToString(hexBinding.getBytes(StandardCharsets.UTF_8));
-                keyAccess.policyBinding = policyBinding;
-
-                // Wrap the key with kas public key
-                AsymEncryption asymmetricEncrypt = new AsymEncryption(kasInfo.PublicKey);
-                byte[] wrappedKey = asymmetricEncrypt.encrypt(symKey);
-
-                keyAccess.wrappedKey = encoder.encodeToString(wrappedKey);
+
 
                 // Add meta data
+                var encryptedMetadata = new String();
                 if(tdfConfig.metaData != null && !tdfConfig.metaData.trim().isEmpty()) {
                     AesGcm aesGcm = new AesGcm(symKey);
                     var encrypted = aesGcm.encrypt(tdfConfig.metaData.getBytes(StandardCharsets.UTF_8));
 
-                    EncryptedMetadata encryptedMetadata = new EncryptedMetadata();
-                    encryptedMetadata.iv = encoder.encodeToString(encrypted.getIv());
-                    encryptedMetadata.ciphertext = encoder.encodeToString(encrypted.asBytes());
+                    EncryptedMetadata em = new EncryptedMetadata();
+                    em.iv = encoder.encodeToString(encrypted.getIv());
+                    em.ciphertext = encoder.encodeToString(encrypted.asBytes());
 
-                    var metadata = gson.toJson(encryptedMetadata);
-                    keyAccess.encryptedMetadata = encoder.encodeToString(metadata.getBytes(StandardCharsets.UTF_8));
+                    var metadata = gson.toJson(em);
+                    encryptedMetadata = encoder.encodeToString(metadata.getBytes(StandardCharsets.UTF_8));
                 }
 
-                symKeys.add(symKey);
-                manifest.encryptionInformation.keyAccessObj.add(keyAccess);
+                for (Config.KASInfo kasInfo: conjunction.get(splitID)){
+                    if (kasInfo.PublicKey == null || kasInfo.PublicKey.isEmpty()) {
+                        throw new KasPublicKeyMissing("Kas public key is missing in kas information list");
+                    }
+
+                    // Wrap the key with kas public key
+                    AsymEncryption asymmetricEncrypt = new AsymEncryption(kasInfo.PublicKey);
+                    byte[] wrappedKey = asymmetricEncrypt.encrypt(symKey);
+
+                    Manifest.KeyAccess keyAccess = new Manifest.KeyAccess();
+                    keyAccess.keyType = kWrapped;
+                    keyAccess.url = kasInfo.URL;
+                    keyAccess.kid = kasInfo.KID;
+                    keyAccess.protocol = kKasProtocol;
+                    keyAccess.policyBinding = policyBinding;
+                    keyAccess.wrappedKey = encoder.encodeToString(wrappedKey);
+                    keyAccess.encryptedMetadata = encryptedMetadata;
+
+                    manifest.encryptionInformation.keyAccessObj.add(keyAccess);
+                }
             }
 
             manifest.encryptionInformation.policy = base64PolicyObject;
@@ -319,10 +380,8 @@ public TDFObject createTDF(InputStream payload,
             throw new KasInfoMissing("kas information is missing");
         }
 
-        fillInPublicKeyInfo(tdfConfig.kasInfoList, kas);
-
         TDFObject tdfObject = new TDFObject();
-        tdfObject.prepareManifest(tdfConfig);
+        tdfObject.prepareManifest(tdfConfig, kas);
 
         long encryptedSegmentSize = tdfConfig.defaultSegmentSize + kGcmIvSize + kAesBlockSize;
         TDFWriter tdfWriter = new TDFWriter(outputStream);
@@ -483,17 +542,6 @@ public TDFObject createTDF(InputStream payload,
         return tdfObject;
     }
 
-    private void fillInPublicKeyInfo(List<Config.KASInfo> kasInfoList, SDK.KAS kas) {
-        for (var kasInfo: kasInfoList) {
-            if (kasInfo.PublicKey != null && !kasInfo.PublicKey.isBlank()) {
-                continue;
-            }
-            logger.info("no public key provided for KAS at {}, retrieving", kasInfo.URL);
-            kasInfo.PublicKey = kas.getPublicKey(kasInfo);
-            kasInfo.KID = kas.getKid(kasInfo);
-        }
-    }
-
     public Reader loadTDF(SeekableByteChannel tdf, Config.AssertionConfig assertionConfig, SDK.KAS kas) throws NotValidateRootSignature, SegmentSizeMismatch,
             IOException, FailedToCreateGMAC, JOSEException, ParseException, NoSuchAlgorithmException, DecoderException {
 
@@ -502,15 +550,39 @@ public Reader loadTDF(SeekableByteChannel tdf, Config.AssertionConfig assertionC
         Manifest manifest = gson.fromJson(manifestJson, Manifest.class);
         byte[] payloadKey = new byte[GCM_KEY_SIZE];
         String unencryptedMetadata = null;
+
+        Set<String> knownSplits = new HashSet<String>();
+        Set<String> foundSplits = new HashSet<String>();;
+        Map<Config.SplitStep, Exception> skippedSplits = new HashMap<>();
+        boolean mixedSplits = manifest.encryptionInformation.keyAccessObj.size() > 1 &&
+         (manifest.encryptionInformation.keyAccessObj.get(0).sid != null) &&
+          !manifest.encryptionInformation.keyAccessObj.get(0).sid.isEmpty();
 
         MessageDigest digest =  MessageDigest.getInstance("SHA-256");
 
         if (manifest.payload.isEncrypted) {
             for (Manifest.KeyAccess keyAccess: manifest.encryptionInformation.keyAccessObj) {
-                var unwrappedKey = kas.unwrap(keyAccess, manifest.encryptionInformation.policy);
+                Config.SplitStep ss = new Config.SplitStep(keyAccess.url, keyAccess.sid);
+                byte[] unwrappedKey;
+                if (!mixedSplits) {
+                    unwrappedKey = kas.unwrap(keyAccess, manifest.encryptionInformation.policy);
+                } else {
+                    knownSplits.add(unencryptedMetadata);
+                    if (foundSplits.contains(ss.splitID)){
+                        continue;
+                    }
+                    try {
+                        unwrappedKey = kas.unwrap(keyAccess, manifest.encryptionInformation.policy);
+                    } catch (Exception e) {
+                        skippedSplits.put(ss, e);
+                        continue;
+                    }
+                }
+
                 for (int index = 0; index < unwrappedKey.length; index++) {
                     payloadKey[index] ^= unwrappedKey[index];
                 }
+                foundSplits.add(ss.splitID);
 
                 if (keyAccess.encryptedMetadata != null && !keyAccess.encryptedMetadata.isEmpty()) {
                     AesGcm aesGcm = new AesGcm(unwrappedKey);
@@ -528,6 +600,22 @@ public Reader loadTDF(SeekableByteChannel tdf, Config.AssertionConfig assertionC
                     unencryptedMetadata = new String(decrypted, StandardCharsets.UTF_8);
                 }
             }
+
+            if (mixedSplits && knownSplits.size() > foundSplits.size()) {
+                List<Exception> exceptionList = new ArrayList<>(skippedSplits.size() + 1);
+                exceptionList.add(new Exception("splitKey.unable to reconstruct split key: " + skippedSplits));
+
+                for (Map.Entry<Config.SplitStep, Exception> entry : skippedSplits.entrySet()) {
+                    exceptionList.add(entry.getValue());
+                }
+
+                StringBuilder combinedMessage = new StringBuilder();
+                for (Exception e : exceptionList) {
+                    combinedMessage.append(e.getMessage()).append("\n");
+                }
+
+                throw new SplitKeyException(combinedMessage.toString());
+            }
         }
 
         // Validate root signature