From 6e035254e02aba17f0b1102ae78c4ff43a664340 Mon Sep 17 00:00:00 2001 From: Kodai Doki Date: Thu, 4 Dec 2025 16:45:11 +0900 Subject: [PATCH 1/8] Fix to increase Jackson stream read constraints to handle large data --- .../db/storage/objectstorage/Serializer.java | 5 ++++ ...DistributedStorageIntegrationTestBase.java | 23 +++++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/core/src/main/java/com/scalar/db/storage/objectstorage/Serializer.java b/core/src/main/java/com/scalar/db/storage/objectstorage/Serializer.java index 93c41822d5..270c6e08c1 100644 --- a/core/src/main/java/com/scalar/db/storage/objectstorage/Serializer.java +++ b/core/src/main/java/com/scalar/db/storage/objectstorage/Serializer.java @@ -1,5 +1,6 @@ package com.scalar.db.storage.objectstorage; +import com.fasterxml.jackson.core.StreamReadConstraints; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.ObjectMapper; @@ -10,6 +11,10 @@ public class Serializer { private static final ObjectMapper mapper = new ObjectMapper(); static { + mapper + .getFactory() + .setStreamReadConstraints( + StreamReadConstraints.builder().maxStringLength(Integer.MAX_VALUE).build()); mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); mapper.configure(SerializationFeature.WRAP_ROOT_VALUE, false); mapper.registerModule(new JavaTimeModule()); diff --git a/integration-test/src/main/java/com/scalar/db/api/DistributedStorageIntegrationTestBase.java b/integration-test/src/main/java/com/scalar/db/api/DistributedStorageIntegrationTestBase.java index 0e6b21e47f..cbe70602da 100644 --- a/integration-test/src/main/java/com/scalar/db/api/DistributedStorageIntegrationTestBase.java +++ b/integration-test/src/main/java/com/scalar/db/api/DistributedStorageIntegrationTestBase.java @@ -1376,6 +1376,29 @@ public void delete_DeleteWithPartitionKeyAndClusteringKeyGiven_ShouldDeleteSingl assertThat(results.get(1).getInt(getColumnName4())).isEqualTo(cKey + 2); } + @Test + public void put_PutWithLargeTextValue_ShouldStoreProperly() throws ExecutionException { + // Arrange + int payloadSizeBytes = 100 * 1024 * 1024; // 100 MiB + StringBuilder builder = new StringBuilder(); + for (long i = 0; i < payloadSizeBytes; i++) { + builder.append("a"); + } + String largeText = builder.toString(); + Put put = preparePuts().get(0); + put = Put.newBuilder(put).textValue(getColumnName2(), largeText).build(); + + // Act + storage.put(put); + + // Assert + Get get = prepareGet(0, 0); + Optional actual = storage.get(get); + assertThat(actual.isPresent()).isTrue(); + assertThat(actual.get().contains(getColumnName2())).isTrue(); + assertThat(actual.get().getText(getColumnName2())).isEqualTo(largeText); + } + @Test public void delete_DeleteWithIfExistsGivenWhenNoSuchRecord_ShouldThrowNoMutationException() { // Arrange From 4dc7f8597cfede916b31ce3c16f861179469dd65 Mon Sep 17 00:00:00 2001 From: Kodai Doki Date: Thu, 4 Dec 2025 17:17:42 +0900 Subject: [PATCH 2/8] Add validation for string length --- .../java/com/scalar/db/common/CoreError.java | 6 ++ .../ObjectStorageOperationChecker.java | 60 +++++++++++++++ .../db/storage/objectstorage/Serializer.java | 3 +- .../ObjectStorageOperationCheckerTest.java | 74 +++++++++++++++++++ 4 files changed, 142 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/com/scalar/db/common/CoreError.java b/core/src/main/java/com/scalar/db/common/CoreError.java index d907f54375..9ce26cf983 100644 --- a/core/src/main/java/com/scalar/db/common/CoreError.java +++ b/core/src/main/java/com/scalar/db/common/CoreError.java @@ -1028,6 +1028,12 @@ public enum CoreError implements ScalarDbError { + "you may be able to adjust the settings to enable consistent reads. Please refer to the storage configuration for details. Storage: %s", "", ""), + OBJECT_STORAGE_EXCEEDS_MAX_VALUE_LENGTH_ALLOWED( + Category.USER_ERROR, + "0279", + "The size of a single column value exceeds the maximum allowed size of %d bytes. Column: %s; Size: %d bytes", + "", + ""), // // Errors for the concurrency error category diff --git a/core/src/main/java/com/scalar/db/storage/objectstorage/ObjectStorageOperationChecker.java b/core/src/main/java/com/scalar/db/storage/objectstorage/ObjectStorageOperationChecker.java index 38322b88c4..2db78cb69e 100644 --- a/core/src/main/java/com/scalar/db/storage/objectstorage/ObjectStorageOperationChecker.java +++ b/core/src/main/java/com/scalar/db/storage/objectstorage/ObjectStorageOperationChecker.java @@ -23,12 +23,71 @@ import com.scalar.db.io.TimeColumn; import com.scalar.db.io.TimestampColumn; import com.scalar.db.io.TimestampTZColumn; +import java.nio.ByteBuffer; public class ObjectStorageOperationChecker extends OperationChecker { private static final char[] ILLEGAL_CHARACTERS_IN_PRIMARY_KEY = { ObjectStorageUtils.OBJECT_KEY_DELIMITER, ObjectStorageUtils.CONCATENATED_KEY_DELIMITER, }; + private static final ColumnVisitor COMMON_COLUMN_CHECKER = + new ColumnVisitor() { + @Override + public void visit(BooleanColumn column) {} + + @Override + public void visit(IntColumn column) {} + + @Override + public void visit(BigIntColumn column) {} + + @Override + public void visit(FloatColumn column) {} + + @Override + public void visit(DoubleColumn column) {} + + @Override + public void visit(TextColumn column) { + String value = column.getTextValue(); + assert value != null; + + if (Serializer.MAX_STRING_LENGTH_ALLOWED < value.length()) { + throw new IllegalArgumentException( + CoreError.OBJECT_STORAGE_EXCEEDS_MAX_VALUE_LENGTH_ALLOWED.buildMessage( + Serializer.MAX_STRING_LENGTH_ALLOWED, column.getName(), value.length())); + } + } + + @Override + public void visit(BlobColumn column) { + ByteBuffer buffer = column.getBlobValue(); + assert buffer != null; + // Calculate the length after Base64 encoding. + int base64EncodedLength = + buffer.capacity() % 3 == 0 + ? (buffer.capacity() / 3) * 4 + : (buffer.capacity() / 3 + 1) * 4; + if (Serializer.MAX_STRING_LENGTH_ALLOWED < base64EncodedLength) { + throw new IllegalArgumentException( + CoreError.OBJECT_STORAGE_EXCEEDS_MAX_VALUE_LENGTH_ALLOWED.buildMessage( + Serializer.MAX_STRING_LENGTH_ALLOWED, column.getName(), base64EncodedLength)); + } + } + + @Override + public void visit(DateColumn column) {} + + @Override + public void visit(TimeColumn column) {} + + @Override + public void visit(TimestampColumn column) {} + + @Override + public void visit(TimestampTZColumn column) {} + }; + private static final ColumnVisitor PRIMARY_KEY_COLUMN_CHECKER = new ColumnVisitor() { @Override @@ -104,6 +163,7 @@ public void check(Scan scan) throws ExecutionException { @Override public void check(Put put) throws ExecutionException { super.check(put); + put.getColumns().values().forEach(column -> column.accept(COMMON_COLUMN_CHECKER)); checkPrimaryKey(put); } diff --git a/core/src/main/java/com/scalar/db/storage/objectstorage/Serializer.java b/core/src/main/java/com/scalar/db/storage/objectstorage/Serializer.java index 270c6e08c1..50de2f515b 100644 --- a/core/src/main/java/com/scalar/db/storage/objectstorage/Serializer.java +++ b/core/src/main/java/com/scalar/db/storage/objectstorage/Serializer.java @@ -8,13 +8,14 @@ import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; public class Serializer { + public static final Integer MAX_STRING_LENGTH_ALLOWED = Integer.MAX_VALUE; private static final ObjectMapper mapper = new ObjectMapper(); static { mapper .getFactory() .setStreamReadConstraints( - StreamReadConstraints.builder().maxStringLength(Integer.MAX_VALUE).build()); + StreamReadConstraints.builder().maxStringLength(MAX_STRING_LENGTH_ALLOWED).build()); mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); mapper.configure(SerializationFeature.WRAP_ROOT_VALUE, false); mapper.registerModule(new JavaTimeModule()); diff --git a/core/src/test/java/com/scalar/db/storage/objectstorage/ObjectStorageOperationCheckerTest.java b/core/src/test/java/com/scalar/db/storage/objectstorage/ObjectStorageOperationCheckerTest.java index 7ab328e823..38575a2739 100644 --- a/core/src/test/java/com/scalar/db/storage/objectstorage/ObjectStorageOperationCheckerTest.java +++ b/core/src/test/java/com/scalar/db/storage/objectstorage/ObjectStorageOperationCheckerTest.java @@ -26,6 +26,8 @@ import com.scalar.db.exception.storage.ExecutionException; import com.scalar.db.io.DataType; import com.scalar.db.io.Key; +import java.lang.reflect.Field; +import java.lang.reflect.Modifier; import java.util.Arrays; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -807,6 +809,78 @@ public void check_ForMutationsWithDeleteWithCondition_ShouldBehaveProperly() .doesNotThrowAnyException(); } + @Test + public void check_PutGiven_WhenTextColumnExceedsMaxLength_ShouldThrowIllegalArgumentException() + throws Exception { + // Arrange + when(metadataManager.getTableMetadata(any())).thenReturn(TABLE_METADATA1); + + // Temporarily set MAX_STRING_LENGTH_ALLOWED to a small value for testing + Field field = Serializer.class.getDeclaredField("MAX_STRING_LENGTH_ALLOWED"); + field.setAccessible(true); + + Field modifiersField = Field.class.getDeclaredField("modifiers"); + modifiersField.setAccessible(true); + modifiersField.setInt(field, field.getModifiers() & ~Modifier.FINAL); + + Integer originalValue = (Integer) field.get(null); + try { + field.set(null, 10); + + Put put = + Put.newBuilder() + .namespace(NAMESPACE_NAME) + .table(TABLE_NAME) + .partitionKey(Key.ofInt(PKEY1, 0)) + .clusteringKey(Key.ofInt(CKEY1, 0)) + .textValue(COL3, "12345678901") // 11 characters, exceeds limit of 10 + .build(); + + // Act Assert + assertThatThrownBy(() -> operationChecker.check(put)) + .isInstanceOf(IllegalArgumentException.class); + } finally { + field.set(null, originalValue); + } + } + + @Test + public void check_PutGiven_WhenBlobColumnExceedsMaxLength_ShouldThrowIllegalArgumentException() + throws Exception { + // Arrange + when(metadataManager.getTableMetadata(any())).thenReturn(TABLE_METADATA1); + + // Temporarily set MAX_STRING_LENGTH_ALLOWED to a small value for testing + Field field = Serializer.class.getDeclaredField("MAX_STRING_LENGTH_ALLOWED"); + field.setAccessible(true); + + Field modifiersField = Field.class.getDeclaredField("modifiers"); + modifiersField.setAccessible(true); + modifiersField.setInt(field, field.getModifiers() & ~Modifier.FINAL); + + Integer originalValue = (Integer) field.get(null); + try { + field.set(null, 10); + + // 9 bytes -> Base64 encoded length = ((9 + 2) / 3) * 4 = 12, which exceeds limit of 10 + byte[] blob = new byte[9]; + Put put = + Put.newBuilder() + .namespace(NAMESPACE_NAME) + .table(TABLE_NAME) + .partitionKey(Key.ofInt(PKEY1, 0)) + .clusteringKey(Key.ofInt(CKEY1, 0)) + .blobValue(COL4, blob) + .build(); + + // Act Assert + assertThatThrownBy(() -> operationChecker.check(put)) + .isInstanceOf(IllegalArgumentException.class); + } finally { + field.set(null, originalValue); + } + } + private Put buildPutWithCondition(MutationCondition condition) { return Put.newBuilder() .namespace(NAMESPACE_NAME) From c1a0917b860fb391bd827f5d8fe75e2e10b438dd Mon Sep 17 00:00:00 2001 From: Kodai Doki Date: Thu, 4 Dec 2025 18:02:13 +0900 Subject: [PATCH 3/8] Apply suggestions from code review --- .../ObjectStorageOperationChecker.java | 13 +++++++------ .../api/DistributedStorageIntegrationTestBase.java | 8 +++----- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/core/src/main/java/com/scalar/db/storage/objectstorage/ObjectStorageOperationChecker.java b/core/src/main/java/com/scalar/db/storage/objectstorage/ObjectStorageOperationChecker.java index 2db78cb69e..ced4f2c01e 100644 --- a/core/src/main/java/com/scalar/db/storage/objectstorage/ObjectStorageOperationChecker.java +++ b/core/src/main/java/com/scalar/db/storage/objectstorage/ObjectStorageOperationChecker.java @@ -50,7 +50,9 @@ public void visit(DoubleColumn column) {} @Override public void visit(TextColumn column) { String value = column.getTextValue(); - assert value != null; + if (value == null) { + return; + } if (Serializer.MAX_STRING_LENGTH_ALLOWED < value.length()) { throw new IllegalArgumentException( @@ -62,12 +64,11 @@ public void visit(TextColumn column) { @Override public void visit(BlobColumn column) { ByteBuffer buffer = column.getBlobValue(); - assert buffer != null; + if (buffer == null) { + return; + } // Calculate the length after Base64 encoding. - int base64EncodedLength = - buffer.capacity() % 3 == 0 - ? (buffer.capacity() / 3) * 4 - : (buffer.capacity() / 3 + 1) * 4; + int base64EncodedLength = ((buffer.capacity() + 2) / 3) * 4; if (Serializer.MAX_STRING_LENGTH_ALLOWED < base64EncodedLength) { throw new IllegalArgumentException( CoreError.OBJECT_STORAGE_EXCEEDS_MAX_VALUE_LENGTH_ALLOWED.buildMessage( diff --git a/integration-test/src/main/java/com/scalar/db/api/DistributedStorageIntegrationTestBase.java b/integration-test/src/main/java/com/scalar/db/api/DistributedStorageIntegrationTestBase.java index cbe70602da..e0b93ac382 100644 --- a/integration-test/src/main/java/com/scalar/db/api/DistributedStorageIntegrationTestBase.java +++ b/integration-test/src/main/java/com/scalar/db/api/DistributedStorageIntegrationTestBase.java @@ -1380,11 +1380,9 @@ public void delete_DeleteWithPartitionKeyAndClusteringKeyGiven_ShouldDeleteSingl public void put_PutWithLargeTextValue_ShouldStoreProperly() throws ExecutionException { // Arrange int payloadSizeBytes = 100 * 1024 * 1024; // 100 MiB - StringBuilder builder = new StringBuilder(); - for (long i = 0; i < payloadSizeBytes; i++) { - builder.append("a"); - } - String largeText = builder.toString(); + char[] chars = new char[payloadSizeBytes]; + Arrays.fill(chars, 'a'); + String largeText = new String(chars); Put put = preparePuts().get(0); put = Put.newBuilder(put).textValue(getColumnName2(), largeText).build(); From 6dd2688ac2716d5aa3066a346cb703492cec4fba Mon Sep 17 00:00:00 2001 From: Kodai Doki Date: Thu, 4 Dec 2025 22:57:30 +0900 Subject: [PATCH 4/8] Add unit tests --- .../ObjectStorageOperationCheckerTest.java | 70 +++++++++++++++++++ ...DistributedStorageIntegrationTestBase.java | 21 ------ 2 files changed, 70 insertions(+), 21 deletions(-) diff --git a/core/src/test/java/com/scalar/db/storage/objectstorage/ObjectStorageOperationCheckerTest.java b/core/src/test/java/com/scalar/db/storage/objectstorage/ObjectStorageOperationCheckerTest.java index 38575a2739..0d1f7ff01b 100644 --- a/core/src/test/java/com/scalar/db/storage/objectstorage/ObjectStorageOperationCheckerTest.java +++ b/core/src/test/java/com/scalar/db/storage/objectstorage/ObjectStorageOperationCheckerTest.java @@ -809,6 +809,76 @@ public void check_ForMutationsWithDeleteWithCondition_ShouldBehaveProperly() .doesNotThrowAnyException(); } + @Test + public void check_PutGiven_WhenTextColumnLengthIsWithinLimit_ShouldNotThrowException() + throws Exception { + // Arrange + when(metadataManager.getTableMetadata(any())).thenReturn(TABLE_METADATA1); + + // Temporarily set MAX_STRING_LENGTH_ALLOWED to a small value for testing + Field field = Serializer.class.getDeclaredField("MAX_STRING_LENGTH_ALLOWED"); + field.setAccessible(true); + + Field modifiersField = Field.class.getDeclaredField("modifiers"); + modifiersField.setAccessible(true); + modifiersField.setInt(field, field.getModifiers() & ~Modifier.FINAL); + + Integer originalValue = (Integer) field.get(null); + try { + field.set(null, 10); + + Put put = + Put.newBuilder() + .namespace(NAMESPACE_NAME) + .table(TABLE_NAME) + .partitionKey(Key.ofInt(PKEY1, 0)) + .clusteringKey(Key.ofInt(CKEY1, 0)) + .textValue(COL3, "1234567890") // 10 characters, exactly at the limit + .build(); + + // Act Assert + assertThatCode(() -> operationChecker.check(put)).doesNotThrowAnyException(); + } finally { + field.set(null, originalValue); + } + } + + @Test + public void check_PutGiven_WhenBlobColumnLengthIsWithinLimit_ShouldNotThrowException() + throws Exception { + // Arrange + when(metadataManager.getTableMetadata(any())).thenReturn(TABLE_METADATA1); + + // Temporarily set MAX_STRING_LENGTH_ALLOWED to a small value for testing + Field field = Serializer.class.getDeclaredField("MAX_STRING_LENGTH_ALLOWED"); + field.setAccessible(true); + + Field modifiersField = Field.class.getDeclaredField("modifiers"); + modifiersField.setAccessible(true); + modifiersField.setInt(field, field.getModifiers() & ~Modifier.FINAL); + + Integer originalValue = (Integer) field.get(null); + try { + field.set(null, 10); + + // 6 bytes -> Base64 encoded length = ((6 + 2) / 3) * 4 = 8, which is within the limit of 10 + byte[] blob = new byte[6]; + Put put = + Put.newBuilder() + .namespace(NAMESPACE_NAME) + .table(TABLE_NAME) + .partitionKey(Key.ofInt(PKEY1, 0)) + .clusteringKey(Key.ofInt(CKEY1, 0)) + .blobValue(COL4, blob) + .build(); + + // Act Assert + assertThatCode(() -> operationChecker.check(put)).doesNotThrowAnyException(); + } finally { + field.set(null, originalValue); + } + } + @Test public void check_PutGiven_WhenTextColumnExceedsMaxLength_ShouldThrowIllegalArgumentException() throws Exception { diff --git a/integration-test/src/main/java/com/scalar/db/api/DistributedStorageIntegrationTestBase.java b/integration-test/src/main/java/com/scalar/db/api/DistributedStorageIntegrationTestBase.java index e0b93ac382..0e6b21e47f 100644 --- a/integration-test/src/main/java/com/scalar/db/api/DistributedStorageIntegrationTestBase.java +++ b/integration-test/src/main/java/com/scalar/db/api/DistributedStorageIntegrationTestBase.java @@ -1376,27 +1376,6 @@ public void delete_DeleteWithPartitionKeyAndClusteringKeyGiven_ShouldDeleteSingl assertThat(results.get(1).getInt(getColumnName4())).isEqualTo(cKey + 2); } - @Test - public void put_PutWithLargeTextValue_ShouldStoreProperly() throws ExecutionException { - // Arrange - int payloadSizeBytes = 100 * 1024 * 1024; // 100 MiB - char[] chars = new char[payloadSizeBytes]; - Arrays.fill(chars, 'a'); - String largeText = new String(chars); - Put put = preparePuts().get(0); - put = Put.newBuilder(put).textValue(getColumnName2(), largeText).build(); - - // Act - storage.put(put); - - // Assert - Get get = prepareGet(0, 0); - Optional actual = storage.get(get); - assertThat(actual.isPresent()).isTrue(); - assertThat(actual.get().contains(getColumnName2())).isTrue(); - assertThat(actual.get().getText(getColumnName2())).isEqualTo(largeText); - } - @Test public void delete_DeleteWithIfExistsGivenWhenNoSuchRecord_ShouldThrowNoMutationException() { // Arrange From 778498e89cc40f75e63c76ffe2a3f337a1c77348 Mon Sep 17 00:00:00 2001 From: Kodai Doki Date: Thu, 4 Dec 2025 23:11:15 +0900 Subject: [PATCH 5/8] Add integration test --- .../ObjectStorageIntegrationTest.java | 30 +++++++++++++++++++ ...DistributedStorageIntegrationTestBase.java | 4 +-- 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/core/src/integration-test/java/com/scalar/db/storage/objectstorage/ObjectStorageIntegrationTest.java b/core/src/integration-test/java/com/scalar/db/storage/objectstorage/ObjectStorageIntegrationTest.java index ace6694d7b..fa602172eb 100644 --- a/core/src/integration-test/java/com/scalar/db/storage/objectstorage/ObjectStorageIntegrationTest.java +++ b/core/src/integration-test/java/com/scalar/db/storage/objectstorage/ObjectStorageIntegrationTest.java @@ -1,11 +1,20 @@ package com.scalar.db.storage.objectstorage; +import static org.assertj.core.api.Assertions.assertThat; + import com.scalar.db.api.DistributedStorageIntegrationTestBase; +import com.scalar.db.api.Get; +import com.scalar.db.api.Put; +import com.scalar.db.api.Result; import com.scalar.db.api.TableMetadata; +import com.scalar.db.exception.storage.ExecutionException; import com.scalar.db.io.DataType; +import java.util.Arrays; import java.util.Map; +import java.util.Optional; import java.util.Properties; import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; public class ObjectStorageIntegrationTest extends DistributedStorageIntegrationTestBase { @@ -57,4 +66,25 @@ public void scan_ScanGivenForIndexedColumn_ShouldScan() {} @Override @Disabled("Object Storage does not support index-related operations") public void scan_ScanGivenForNonIndexedColumn_ShouldThrowIllegalArgumentException() {} + + @Test + public void put_PutWithLargeTextValue_ShouldStoreProperly() throws ExecutionException { + // Arrange + int payloadSizeBytes = 20000000; // The default string length limit of Jackson + char[] chars = new char[payloadSizeBytes + 1]; + Arrays.fill(chars, 'a'); + String largeText = new String(chars); + Put put = preparePuts().get(0); + put = Put.newBuilder(put).textValue(getColumnName2(), largeText).build(); + + // Act + storage.put(put); + + // Assert + Get get = prepareGet(0, 0); + Optional actual = storage.get(get); + assertThat(actual.isPresent()).isTrue(); + assertThat(actual.get().contains(getColumnName2())).isTrue(); + assertThat(actual.get().getText(getColumnName2())).isEqualTo(largeText); + } } diff --git a/integration-test/src/main/java/com/scalar/db/api/DistributedStorageIntegrationTestBase.java b/integration-test/src/main/java/com/scalar/db/api/DistributedStorageIntegrationTestBase.java index 0e6b21e47f..c166a002f1 100644 --- a/integration-test/src/main/java/com/scalar/db/api/DistributedStorageIntegrationTestBase.java +++ b/integration-test/src/main/java/com/scalar/db/api/DistributedStorageIntegrationTestBase.java @@ -2396,7 +2396,7 @@ private void populateRecords() { puts.forEach(p -> assertThatCode(() -> storage.put(p)).doesNotThrowAnyException()); } - private Get prepareGet(int pKey, int cKey) { + protected Get prepareGet(int pKey, int cKey) { Key partitionKey = Key.ofInt(getColumnName1(), pKey); Key clusteringKey = Key.ofInt(getColumnName4(), cKey); return Get.newBuilder() @@ -2407,7 +2407,7 @@ private Get prepareGet(int pKey, int cKey) { .build(); } - private List preparePuts() { + protected List preparePuts() { List puts = new ArrayList<>(); IntStream.range(0, 5) From 5f6d51829864038ef6620fb86da93391707cd944 Mon Sep 17 00:00:00 2001 From: Kodai Doki Date: Fri, 5 Dec 2025 09:28:07 +0900 Subject: [PATCH 6/8] Apply suggestions from code review --- .../java/com/scalar/db/common/CoreError.java | 4 +- .../ObjectStorageOperationChecker.java | 21 +-- .../db/storage/objectstorage/Serializer.java | 2 +- .../ObjectStorageOperationCheckerTest.java | 172 +++++------------- 4 files changed, 51 insertions(+), 148 deletions(-) diff --git a/core/src/main/java/com/scalar/db/common/CoreError.java b/core/src/main/java/com/scalar/db/common/CoreError.java index 9ce26cf983..d93f19d182 100644 --- a/core/src/main/java/com/scalar/db/common/CoreError.java +++ b/core/src/main/java/com/scalar/db/common/CoreError.java @@ -1028,10 +1028,10 @@ public enum CoreError implements ScalarDbError { + "you may be able to adjust the settings to enable consistent reads. Please refer to the storage configuration for details. Storage: %s", "", ""), - OBJECT_STORAGE_EXCEEDS_MAX_VALUE_LENGTH_ALLOWED( + OBJECT_STORAGE_BLOB_EXCEEDS_MAX_LENGTH_ALLOWED( Category.USER_ERROR, "0279", - "The size of a single column value exceeds the maximum allowed size of %d bytes. Column: %s; Size: %d bytes", + "The size of a BLOB column value exceeds the maximum allowed size of %d bytes. Column: %s; Size: %d bytes", "", ""), diff --git a/core/src/main/java/com/scalar/db/storage/objectstorage/ObjectStorageOperationChecker.java b/core/src/main/java/com/scalar/db/storage/objectstorage/ObjectStorageOperationChecker.java index ced4f2c01e..e6c3a791fd 100644 --- a/core/src/main/java/com/scalar/db/storage/objectstorage/ObjectStorageOperationChecker.java +++ b/core/src/main/java/com/scalar/db/storage/objectstorage/ObjectStorageOperationChecker.java @@ -48,18 +48,7 @@ public void visit(FloatColumn column) {} public void visit(DoubleColumn column) {} @Override - public void visit(TextColumn column) { - String value = column.getTextValue(); - if (value == null) { - return; - } - - if (Serializer.MAX_STRING_LENGTH_ALLOWED < value.length()) { - throw new IllegalArgumentException( - CoreError.OBJECT_STORAGE_EXCEEDS_MAX_VALUE_LENGTH_ALLOWED.buildMessage( - Serializer.MAX_STRING_LENGTH_ALLOWED, column.getName(), value.length())); - } - } + public void visit(TextColumn column) {} @Override public void visit(BlobColumn column) { @@ -68,11 +57,11 @@ public void visit(BlobColumn column) { return; } // Calculate the length after Base64 encoding. - int base64EncodedLength = ((buffer.capacity() + 2) / 3) * 4; - if (Serializer.MAX_STRING_LENGTH_ALLOWED < base64EncodedLength) { + int allowedLength = Serializer.MAX_STRING_LENGTH_ALLOWED / 4 * 3; + if (buffer.remaining() > allowedLength) { throw new IllegalArgumentException( - CoreError.OBJECT_STORAGE_EXCEEDS_MAX_VALUE_LENGTH_ALLOWED.buildMessage( - Serializer.MAX_STRING_LENGTH_ALLOWED, column.getName(), base64EncodedLength)); + CoreError.OBJECT_STORAGE_BLOB_EXCEEDS_MAX_LENGTH_ALLOWED.buildMessage( + Serializer.MAX_STRING_LENGTH_ALLOWED, column.getName(), buffer.remaining())); } } diff --git a/core/src/main/java/com/scalar/db/storage/objectstorage/Serializer.java b/core/src/main/java/com/scalar/db/storage/objectstorage/Serializer.java index 50de2f515b..9e9cb346a9 100644 --- a/core/src/main/java/com/scalar/db/storage/objectstorage/Serializer.java +++ b/core/src/main/java/com/scalar/db/storage/objectstorage/Serializer.java @@ -8,7 +8,7 @@ import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; public class Serializer { - public static final Integer MAX_STRING_LENGTH_ALLOWED = Integer.MAX_VALUE; + public static final int MAX_STRING_LENGTH_ALLOWED = Integer.MAX_VALUE; private static final ObjectMapper mapper = new ObjectMapper(); static { diff --git a/core/src/test/java/com/scalar/db/storage/objectstorage/ObjectStorageOperationCheckerTest.java b/core/src/test/java/com/scalar/db/storage/objectstorage/ObjectStorageOperationCheckerTest.java index 0d1f7ff01b..6bdbd2e599 100644 --- a/core/src/test/java/com/scalar/db/storage/objectstorage/ObjectStorageOperationCheckerTest.java +++ b/core/src/test/java/com/scalar/db/storage/objectstorage/ObjectStorageOperationCheckerTest.java @@ -9,6 +9,8 @@ import static org.assertj.core.api.Assertions.assertThatCode; import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; import static org.mockito.Mockito.when; import static org.mockito.MockitoAnnotations.openMocks; @@ -24,11 +26,14 @@ import com.scalar.db.common.TableMetadataManager; import com.scalar.db.config.DatabaseConfig; import com.scalar.db.exception.storage.ExecutionException; +import com.scalar.db.io.BlobColumn; +import com.scalar.db.io.Column; import com.scalar.db.io.DataType; import com.scalar.db.io.Key; -import java.lang.reflect.Field; -import java.lang.reflect.Modifier; +import java.nio.ByteBuffer; import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.Map; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.mockito.Mock; @@ -810,145 +815,54 @@ public void check_ForMutationsWithDeleteWithCondition_ShouldBehaveProperly() } @Test - public void check_PutGiven_WhenTextColumnLengthIsWithinLimit_ShouldNotThrowException() - throws Exception { + public void check_PutGiven_WhenBlobColumnIsWithinLimit_ShouldNotThrowException() + throws ExecutionException { // Arrange when(metadataManager.getTableMetadata(any())).thenReturn(TABLE_METADATA1); - // Temporarily set MAX_STRING_LENGTH_ALLOWED to a small value for testing - Field field = Serializer.class.getDeclaredField("MAX_STRING_LENGTH_ALLOWED"); - field.setAccessible(true); - - Field modifiersField = Field.class.getDeclaredField("modifiers"); - modifiersField.setAccessible(true); - modifiersField.setInt(field, field.getModifiers() & ~Modifier.FINAL); - - Integer originalValue = (Integer) field.get(null); - try { - field.set(null, 10); - - Put put = - Put.newBuilder() - .namespace(NAMESPACE_NAME) - .table(TABLE_NAME) - .partitionKey(Key.ofInt(PKEY1, 0)) - .clusteringKey(Key.ofInt(CKEY1, 0)) - .textValue(COL3, "1234567890") // 10 characters, exactly at the limit - .build(); - - // Act Assert - assertThatCode(() -> operationChecker.check(put)).doesNotThrowAnyException(); - } finally { - field.set(null, originalValue); - } - } - - @Test - public void check_PutGiven_WhenBlobColumnLengthIsWithinLimit_ShouldNotThrowException() - throws Exception { - // Arrange - when(metadataManager.getTableMetadata(any())).thenReturn(TABLE_METADATA1); + byte[] blob = new byte[100]; + Put put = + Put.newBuilder() + .namespace(NAMESPACE_NAME) + .table(TABLE_NAME) + .partitionKey(Key.ofInt(PKEY1, 0)) + .clusteringKey(Key.ofInt(CKEY1, 0)) + .blobValue(COL4, blob) + .build(); - // Temporarily set MAX_STRING_LENGTH_ALLOWED to a small value for testing - Field field = Serializer.class.getDeclaredField("MAX_STRING_LENGTH_ALLOWED"); - field.setAccessible(true); - - Field modifiersField = Field.class.getDeclaredField("modifiers"); - modifiersField.setAccessible(true); - modifiersField.setInt(field, field.getModifiers() & ~Modifier.FINAL); - - Integer originalValue = (Integer) field.get(null); - try { - field.set(null, 10); - - // 6 bytes -> Base64 encoded length = ((6 + 2) / 3) * 4 = 8, which is within the limit of 10 - byte[] blob = new byte[6]; - Put put = - Put.newBuilder() - .namespace(NAMESPACE_NAME) - .table(TABLE_NAME) - .partitionKey(Key.ofInt(PKEY1, 0)) - .clusteringKey(Key.ofInt(CKEY1, 0)) - .blobValue(COL4, blob) - .build(); - - // Act Assert - assertThatCode(() -> operationChecker.check(put)).doesNotThrowAnyException(); - } finally { - field.set(null, originalValue); - } + // Act Assert + assertThatCode(() -> operationChecker.check(put)).doesNotThrowAnyException(); } @Test - public void check_PutGiven_WhenTextColumnExceedsMaxLength_ShouldThrowIllegalArgumentException() - throws Exception { + public void check_PutGiven_WhenBlobColumnExceedsLimit_ShouldThrowIllegalArgumentException() + throws ExecutionException { // Arrange when(metadataManager.getTableMetadata(any())).thenReturn(TABLE_METADATA1); - // Temporarily set MAX_STRING_LENGTH_ALLOWED to a small value for testing - Field field = Serializer.class.getDeclaredField("MAX_STRING_LENGTH_ALLOWED"); - field.setAccessible(true); - - Field modifiersField = Field.class.getDeclaredField("modifiers"); - modifiersField.setAccessible(true); - modifiersField.setInt(field, field.getModifiers() & ~Modifier.FINAL); - - Integer originalValue = (Integer) field.get(null); - try { - field.set(null, 10); - - Put put = - Put.newBuilder() - .namespace(NAMESPACE_NAME) - .table(TABLE_NAME) - .partitionKey(Key.ofInt(PKEY1, 0)) - .clusteringKey(Key.ofInt(CKEY1, 0)) - .textValue(COL3, "12345678901") // 11 characters, exceeds limit of 10 - .build(); - - // Act Assert - assertThatThrownBy(() -> operationChecker.check(put)) - .isInstanceOf(IllegalArgumentException.class); - } finally { - field.set(null, originalValue); - } - } + int allowedLength = Serializer.MAX_STRING_LENGTH_ALLOWED / 4 * 3; + ByteBuffer mockBuffer = mock(ByteBuffer.class); + when(mockBuffer.remaining()).thenReturn(allowedLength + 1); - @Test - public void check_PutGiven_WhenBlobColumnExceedsMaxLength_ShouldThrowIllegalArgumentException() - throws Exception { - // Arrange - when(metadataManager.getTableMetadata(any())).thenReturn(TABLE_METADATA1); + BlobColumn blobColumn = mock(BlobColumn.class); + when(blobColumn.getName()).thenReturn(COL4); + when(blobColumn.getBlobValue()).thenReturn(mockBuffer); + + Put put = + spy( + Put.newBuilder() + .namespace(NAMESPACE_NAME) + .table(TABLE_NAME) + .partitionKey(Key.ofInt(PKEY1, 0)) + .clusteringKey(Key.ofInt(CKEY1, 0)) + .build()); + Map> columns = new LinkedHashMap<>(); + columns.put(COL4, blobColumn); + when(put.getColumns()).thenReturn(columns); - // Temporarily set MAX_STRING_LENGTH_ALLOWED to a small value for testing - Field field = Serializer.class.getDeclaredField("MAX_STRING_LENGTH_ALLOWED"); - field.setAccessible(true); - - Field modifiersField = Field.class.getDeclaredField("modifiers"); - modifiersField.setAccessible(true); - modifiersField.setInt(field, field.getModifiers() & ~Modifier.FINAL); - - Integer originalValue = (Integer) field.get(null); - try { - field.set(null, 10); - - // 9 bytes -> Base64 encoded length = ((9 + 2) / 3) * 4 = 12, which exceeds limit of 10 - byte[] blob = new byte[9]; - Put put = - Put.newBuilder() - .namespace(NAMESPACE_NAME) - .table(TABLE_NAME) - .partitionKey(Key.ofInt(PKEY1, 0)) - .clusteringKey(Key.ofInt(CKEY1, 0)) - .blobValue(COL4, blob) - .build(); - - // Act Assert - assertThatThrownBy(() -> operationChecker.check(put)) - .isInstanceOf(IllegalArgumentException.class); - } finally { - field.set(null, originalValue); - } + // Act Assert + assertThatThrownBy(() -> operationChecker.check(put)) + .isInstanceOf(IllegalArgumentException.class); } private Put buildPutWithCondition(MutationCondition condition) { From 9a3e00d51febd53afa0995e8bfe6fd84a4be3690 Mon Sep 17 00:00:00 2001 From: Kodai Doki Date: Fri, 5 Dec 2025 09:52:15 +0900 Subject: [PATCH 7/8] Remove integraition test --- .../ObjectStorageIntegrationTest.java | 30 ------------------- 1 file changed, 30 deletions(-) diff --git a/core/src/integration-test/java/com/scalar/db/storage/objectstorage/ObjectStorageIntegrationTest.java b/core/src/integration-test/java/com/scalar/db/storage/objectstorage/ObjectStorageIntegrationTest.java index fa602172eb..ace6694d7b 100644 --- a/core/src/integration-test/java/com/scalar/db/storage/objectstorage/ObjectStorageIntegrationTest.java +++ b/core/src/integration-test/java/com/scalar/db/storage/objectstorage/ObjectStorageIntegrationTest.java @@ -1,20 +1,11 @@ package com.scalar.db.storage.objectstorage; -import static org.assertj.core.api.Assertions.assertThat; - import com.scalar.db.api.DistributedStorageIntegrationTestBase; -import com.scalar.db.api.Get; -import com.scalar.db.api.Put; -import com.scalar.db.api.Result; import com.scalar.db.api.TableMetadata; -import com.scalar.db.exception.storage.ExecutionException; import com.scalar.db.io.DataType; -import java.util.Arrays; import java.util.Map; -import java.util.Optional; import java.util.Properties; import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Test; public class ObjectStorageIntegrationTest extends DistributedStorageIntegrationTestBase { @@ -66,25 +57,4 @@ public void scan_ScanGivenForIndexedColumn_ShouldScan() {} @Override @Disabled("Object Storage does not support index-related operations") public void scan_ScanGivenForNonIndexedColumn_ShouldThrowIllegalArgumentException() {} - - @Test - public void put_PutWithLargeTextValue_ShouldStoreProperly() throws ExecutionException { - // Arrange - int payloadSizeBytes = 20000000; // The default string length limit of Jackson - char[] chars = new char[payloadSizeBytes + 1]; - Arrays.fill(chars, 'a'); - String largeText = new String(chars); - Put put = preparePuts().get(0); - put = Put.newBuilder(put).textValue(getColumnName2(), largeText).build(); - - // Act - storage.put(put); - - // Assert - Get get = prepareGet(0, 0); - Optional actual = storage.get(get); - assertThat(actual.isPresent()).isTrue(); - assertThat(actual.get().contains(getColumnName2())).isTrue(); - assertThat(actual.get().getText(getColumnName2())).isEqualTo(largeText); - } } From e00b5112085b483942ffba22639b0c5f03819f6c Mon Sep 17 00:00:00 2001 From: Kodai Doki Date: Sat, 6 Dec 2025 11:40:59 +0900 Subject: [PATCH 8/8] Apply suggestions from code review --- .../storage/objectstorage/ObjectStorageOperationChecker.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/com/scalar/db/storage/objectstorage/ObjectStorageOperationChecker.java b/core/src/main/java/com/scalar/db/storage/objectstorage/ObjectStorageOperationChecker.java index e6c3a791fd..30849ffb55 100644 --- a/core/src/main/java/com/scalar/db/storage/objectstorage/ObjectStorageOperationChecker.java +++ b/core/src/main/java/com/scalar/db/storage/objectstorage/ObjectStorageOperationChecker.java @@ -56,8 +56,8 @@ public void visit(BlobColumn column) { if (buffer == null) { return; } - // Calculate the length after Base64 encoding. - int allowedLength = Serializer.MAX_STRING_LENGTH_ALLOWED / 4 * 3; + // Calculate the maximum allowed blob length after Base64 encoding. + long allowedLength = (long) Serializer.MAX_STRING_LENGTH_ALLOWED / 4 * 3; if (buffer.remaining() > allowedLength) { throw new IllegalArgumentException( CoreError.OBJECT_STORAGE_BLOB_EXCEEDS_MAX_LENGTH_ALLOWED.buildMessage(