perf: usage of custom collections/maps in hot paths

Fixed issue #3965

Author: lvca

engine/src/main/java/com/arcadedb/database/LocalTransactionExplicitLock.java

@@ -23,18 +23,18 @@
 import com.arcadedb.index.IndexInternal;
 import com.arcadedb.log.LogManager;
 import com.arcadedb.schema.DocumentType;
+import com.arcadedb.utility.IntHashSet;
 
-import java.util.*;
 import java.util.logging.Level;
 
 /**
  * Explicit lock on a transaction to lock buckets and types in pessimistic way. This avoids the retry mechanism of default implicit locking.
  *
- * @author Luca Garulli (l.garulli@arcadedata.com)
+ * @author Luca Garulli (l.garulli--(at)--arcadedata.com)
  */
 public class LocalTransactionExplicitLock implements TransactionExplicitLock {
   private final TransactionContext transactionContext;
-  private final Set<Integer>       filesToLock = new HashSet<>();
+  private final IntHashSet         filesToLock = new IntHashSet();
 
   public LocalTransactionExplicitLock(final TransactionContext transactionContext) {
     this.transactionContext = transactionContext;
@@ -43,15 +43,13 @@ public LocalTransactionExplicitLock(final TransactionContext transactionContext)
   @Override
   public LocalTransactionExplicitLock bucket(final String bucketName) {
     final Bucket bucket = transactionContext.getDatabase().getSchema().getBucketByName(bucketName);
-    filesToLock.add(bucket.getFileId());
+    addNonNegative(bucket.getFileId());
     final DocumentType associatedType = transactionContext.getDatabase().getSchema().getInvolvedTypeByBucketId(bucket.getFileId());
     if (associatedType != null)
-      filesToLock.addAll(associatedType.getAllIndexes(true).stream()
-          .flatMap(i -> Arrays.stream(i.getIndexesOnBuckets()))
-          .map(b -> b.getFileId())
-          .toList());
+      for (final var typeIndex : associatedType.getAllIndexes(true))
+        for (final IndexInternal idx : typeIndex.getIndexesOnBuckets())
+          addNonNegative(idx.getFileId());
 
-    filesToLock.removeIf((f) -> f < 0); // Remove negative file IDs (e.g., for virtual buckets)
     return this;
   }
 
@@ -60,19 +58,18 @@ public LocalTransactionExplicitLock type(final String typeName) {
     final DocumentType type = transactionContext.getDatabase().getSchema().getType(typeName);
 
     // Lock all indexes for this type
-    filesToLock.addAll(type.getAllIndexes(true).stream()
-        .flatMap(i -> Arrays.stream(i.getIndexesOnBuckets()))
-        .map(IndexInternal::getFileId)
-        .toList());
+    for (final var typeIndex : type.getAllIndexes(true))
+      for (final IndexInternal idx : typeIndex.getIndexesOnBuckets())
+        addNonNegative(idx.getFileId());
 
     // Lock all currently involved buckets for this type
-    filesToLock.addAll(type.getInvolvedBuckets().stream().map(b -> b.getFileId()).toList());
+    for (final Bucket b : type.getInvolvedBuckets())
+      addNonNegative(b.getFileId());
 
     // COMPREHENSIVE BUCKET LOCKING: Also lock all polymorphic buckets (includes subtypes)
     // This helps handle cases where records might be created in buckets not initially involved
-    filesToLock.addAll(type.getBuckets(true).stream().map(b -> b.getFileId()).toList());
-
-    filesToLock.removeIf((f) -> f < 0); // Remove negative file IDs (e.g., for virtual buckets)
+    for (final Bucket b : type.getBuckets(true))
+      addNonNegative(b.getFileId());
 
     LogManager.instance().log(this, Level.FINE,
       "Explicit lock for type '%s' will lock %d bucket files (threadId=%d)",
@@ -85,4 +82,9 @@ public LocalTransactionExplicitLock type(final String typeName) {
   public void lock() {
     transactionContext.explicitLock(filesToLock);
   }
+
+  private void addNonNegative(final int fileId) {
+    if (fileId >= 0)
+      filesToLock.add(fileId);
+  }
 }

engine/src/main/java/com/arcadedb/database/TransactionContext.java

@@ -38,6 +38,7 @@
 import com.arcadedb.index.lsm.LSMTreeIndexAbstract;
 import com.arcadedb.log.LogManager;
 import com.arcadedb.schema.LocalSchema;
+import com.arcadedb.utility.IntHashSet;
 import com.arcadedb.utility.RidHashSet;
 
 import java.io.*;
@@ -545,8 +546,11 @@ public void commitFromReplica(final WALFile.WALTransaction buffer,
     }
 
     try {
-      // LOCK FILES (IN ORDER, SO TO AVOID DEADLOCK)
-      final Set<Integer> modifiedFiles = new HashSet<>();
+      // LOCK FILES (IN ORDER, SO TO AVOID DEADLOCK).
+      // IntHashSet (zero-boxing) is used because this set is built on every commit
+      // and the small Integer boxing churn shows up in young-gen GC pressure under
+      // high transaction throughput.
+      final IntHashSet modifiedFiles = new IntHashSet(buffer.pages.length + 4);
 
       for (final WALFile.WALPage p : buffer.pages)
         modifiedFiles.add(p.fileId);
@@ -620,7 +624,7 @@ public TransactionPhase1 commit1stPhase(final boolean isLeader) {
             ((LocalBucket) database.getSchema().getBucketById(rid.getBucketId())).fetchPageInTransaction(rid);
           }
 
-        final Set<Integer> modifiedFiles = lockFilesFromChanges();
+        final IntHashSet modifiedFiles = lockFilesFromChanges();
 
         if (explicitLockedFiles != null)
           checkExplicitLocks(modifiedFiles);
@@ -866,7 +870,7 @@ public void setStatus(final STATUS status) {
     this.status = status;
   }
 
-  protected void explicitLock(final Set<Integer> filesToLock) {
+  protected void explicitLock(final IntHashSet filesToLock) {
     if (explicitLockedFiles != null)
       throw new TransactionException("Explicit lock already acquired");
 
@@ -879,8 +883,8 @@ protected void explicitLock(final Set<Integer> filesToLock) {
     explicitLockedFiles = lockFilesInOrder(filesToLock);
   }
 
-  private Set<Integer> lockFilesFromChanges() {
-    final Set<Integer> modifiedFiles = new HashSet<>();
+  private IntHashSet lockFilesFromChanges() {
+    final IntHashSet modifiedFiles = new IntHashSet(modifiedPages.size() + 16);
 
     for (final PageId p : modifiedPages.keySet())
       modifiedFiles.add(p.getFileId());
@@ -890,15 +894,16 @@ private Set<Integer> lockFilesFromChanges() {
 
     indexChanges.addFilesToLock(modifiedFiles);
 
-    modifiedFiles.addAll(newPageCounters.keySet());
+    for (final Integer fid : newPageCounters.keySet())
+      modifiedFiles.add(fid);
 
     return modifiedFiles;
   }
 
-  private List<Integer> lockFilesInOrder(final Set<Integer> files) {
+  private List<Integer> lockFilesInOrder(final IntHashSet files) {
     final long timeout = database.getConfiguration().getValueAsLong(GlobalConfiguration.COMMIT_LOCK_TIMEOUT);
 
-    final List<Integer> locked = database.getTransactionManager().tryLockFiles(files, timeout, getRequester());
+    final List<Integer> locked = database.getTransactionManager().tryLockFiles(files.toArray(), timeout, getRequester());
 
     // CHECK IF ALL THE LOCKED FILES STILL EXIST. FILE MISSING CAN HAPPEN IN CASE OF INDEX COMPACTION OR DROP OF A BUCKET OR AN INDEX
     for (Integer f : locked)
@@ -915,9 +920,14 @@ private List<Integer> lockFilesInOrder(final Set<Integer> files) {
     return locked;
   }
 
-  private void checkExplicitLocks(final Set<Integer> modifiedFiles) {
-    // CHECK THE LOCKED FILES ARE ALL LOCKED ALREADY
-    if (!explicitLockedFiles.containsAll(modifiedFiles)) {
+  private void checkExplicitLocks(final IntHashSet modifiedFiles) {
+    // CHECK THE LOCKED FILES ARE ALL LOCKED ALREADY: every modified file must already be in explicitLockedFiles.
+    final boolean[] missing = { false };
+    modifiedFiles.forEach(fid -> {
+      if (!missing[0] && !explicitLockedFiles.contains(fid))
+        missing[0] = true;
+    });
+    if (missing[0]) {
       boolean anyMigration = false;
       // CHECK FOR ANY MIGRATED FILES (INDEX COMPACTION)
       final List<Integer> migratedFileIds = new ArrayList<>(explicitLockedFiles.size());
@@ -932,13 +942,23 @@ private void checkExplicitLocks(final Set<Integer> modifiedFiles) {
         }
       }
 
-      if (anyMigration && migratedFileIds.containsAll(modifiedFiles))
-        // FOUND MIGRATED FILE(S), FORCE THE CLIENT TO RETRY THE OPERATION
-        throw new ConcurrentModificationException(
-            "Error on commit transaction: some files have been migrated, please retry the operation");
+      if (anyMigration) {
+        // Check if EVERY modifiedFiles entry is in migratedFileIds
+        final HashSet<Integer> migratedSet = new HashSet<>(migratedFileIds);
+        final boolean[] allMigrated = { true };
+        modifiedFiles.forEach(fid -> {
+          if (allMigrated[0] && !migratedSet.contains(fid))
+            allMigrated[0] = false;
+        });
+        if (allMigrated[0])
+          // FOUND MIGRATED FILE(S), FORCE THE CLIENT TO RETRY THE OPERATION
+          throw new ConcurrentModificationException(
+              "Error on commit transaction: some files have been migrated, please retry the operation");
+      }
 
       // ERROR: NOT ALL THE MODIFIED FILES ARE LOCKED
-      final HashSet<Integer> left = new HashSet<>(modifiedFiles);
+      final HashSet<Integer> left = new HashSet<>(modifiedFiles.size());
+      modifiedFiles.forEach(left::add);
       left.removeAll(explicitLockedFiles);
 
       final Set<String> resourceNames = left.stream().map((fileId -> database.getSchema().getFileById(fileId).getName()))

engine/src/main/java/com/arcadedb/database/TransactionIndexContext.java

@@ -31,6 +31,7 @@
 import com.arcadedb.schema.Schema;
 import com.arcadedb.serializer.BinaryComparator;
 import com.arcadedb.utility.CollectionUtils;
+import com.arcadedb.utility.IntHashSet;
 
 import java.util.*;
 import java.util.logging.*;
@@ -244,7 +245,7 @@ else if (key.operation == IndexKey.IndexKeyOperation.REPLACE && key.oldRid != nu
     indexEntries.clear();
   }
 
-  public void addFilesToLock(final Set<Integer> modifiedFiles) {
+  public void addFilesToLock(final IntHashSet modifiedFiles) {
     final Schema schema = database.getSchema();
 
     final Set<Index> lockedIndexes = new HashSet<>(indexEntries.size());
@@ -266,7 +267,8 @@ public void addFilesToLock(final Set<Integer> modifiedFiles) {
         // LOCK ALL THE FILES IMPACTED BY THE INDEX KEYS TO CHECK FOR UNIQUE CONSTRAINT
         // TODO: OPTIMIZE LOCKING IF STRATEGY IS PARTITIONED: LOCK ONLY THE RELEVANT INDEX
         final DocumentType type = schema.getType(index.getTypeName());
-        modifiedFiles.addAll(type.getBucketIds(false));
+        for (final int bid : type.getBucketIds(false))
+          modifiedFiles.add(bid);
 
         for (final TypeIndex typeIndex : type.getAllIndexes(true))
           for (final IndexInternal idx : typeIndex.getIndexesOnBuckets())

engine/src/main/java/com/arcadedb/engine/TransactionManager.java

@@ -517,6 +517,41 @@ else if (lock == LockManager.LOCK_STATUS.NO) {
     return lockedFiles;
   }
 
+  /**
+   * Primitive-array overload to avoid Integer boxing on the per-commit hot path. The
+   * passed-in {@code fileIds} array is sorted in place to acquire locks in deadlock-safe order.
+   */
+  public List<Integer> tryLockFiles(final int[] fileIds, final long timeout, final Object requester) {
+    // ORDER THE FILES TO AVOID DEADLOCK
+    Arrays.sort(fileIds);
+
+    final List<Integer> lockedFiles = new ArrayList<>(fileIds.length);
+
+    int attemptFileId = -1;
+    for (final int fileId : fileIds) {
+      attemptFileId = fileId;
+
+      final LockManager.LOCK_STATUS lock = tryLockFile(fileId, timeout, requester);
+
+      if (lock == LockManager.LOCK_STATUS.YES)
+        lockedFiles.add(fileId);
+      else if (lock == LockManager.LOCK_STATUS.NO) {
+        // ERROR: UNLOCK LOCKED FILES
+        unlockFilesInOrder(lockedFiles, requester);
+
+        throw new TimeoutException(
+            "Timeout on locking file " + attemptFileId + " (" + database.getFileManager().getFile(attemptFileId).getFileName()
+                + ") during commit (fileIds=" + Arrays.toString(fileIds) + ", timeout=" + timeout + "ms");
+      }
+    }
+
+    // OK: ALL LOCKED
+    if (LogManager.instance().isDebugEnabled())
+      LogManager.instance().log(this, Level.FINE, "Locked files %s (threadId=%d)", null, Arrays.toString(fileIds),
+          Thread.currentThread().threadId());
+    return lockedFiles;
+  }
+
   public void unlockFilesInOrder(final List<Integer> lockedFileIds, final Object requester) {
     if (lockedFileIds != null && !lockedFileIds.isEmpty()) {
       for (final Integer fileId : lockedFileIds)

engine/src/main/java/com/arcadedb/function/sql/vector/SQLFunctionVectorNeighbors.java

@@ -33,6 +33,7 @@
 import com.arcadedb.query.sql.executor.CommandContext;
 import com.arcadedb.query.sql.executor.ResultSet;
 import com.arcadedb.schema.DocumentType;
+import com.arcadedb.utility.IntHashSet;
 import com.arcadedb.utility.Pair;
 
 import java.util.*;
@@ -113,8 +114,10 @@ public Object execute(final Object self, final Identifiable currentRecord, final
           "No vector index found on property '" + propertyName + "' for type '" + specifiedTypeName + "' or its parent types");
     }
 
-    // Get the bucket IDs that belong to the specified type (not polymorphic - just this type's own buckets)
-    final Set<Integer> allowedBucketIds = new HashSet<>();
+    // Get the bucket IDs that belong to the specified type (not polymorphic - just this type's own buckets).
+    // IntHashSet is zero-boxing - the contains() in executeWithTypeIndex runs once per bucket index per
+    // vector query, and avoiding Integer boxing on every probe matters under sustained ANN workloads.
+    final IntHashSet allowedBucketIds = new IntHashSet();
     for (final Bucket bucket : specifiedType.getBuckets(false)) {
       allowedBucketIds.add(bucket.getFileId());
     }
@@ -144,7 +147,7 @@ else if (item instanceof String s)
     return out;
   }
 
-  private Object executeWithTypeIndex(final TypeIndex typeIndex, final Set<Integer> allowedBucketIds, final Object key,
+  private Object executeWithTypeIndex(final TypeIndex typeIndex, final IntHashSet allowedBucketIds, final Object key,
       final int limit, final int efSearch, final Set<RID> allowedRIDs, final CommandContext context) {
     final var bucketIndexes = typeIndex.getIndexesOnBuckets();
     if (bucketIndexes == null || bucketIndexes.length == 0) {

engine/src/main/java/com/arcadedb/graph/GraphBatch.java

@@ -34,6 +34,7 @@
 import com.arcadedb.schema.Property;
 import com.arcadedb.serializer.BinarySerializer;
 import com.arcadedb.serializer.BinaryTypes;
+import com.arcadedb.utility.LongHashSet;
 
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -755,15 +756,17 @@ private void batchUpdateVertexHeadChunks() {
 
     final long startNs = System.nanoTime();
 
-    // Collect all vertex keys that need updating
-    final Set<Long> allKeys = new HashSet<>(deferredOutHead.keySet());
-    allKeys.addAll(deferredInHead.keySet());
+    // Collect all vertex keys that need updating. Using LongHashSet (zero-boxing,
+    // open-addressing) instead of HashSet<Long> avoids ~70 bytes/entry of overhead
+    // and Long boxing on every addAll, which matters during 100K+ entry bulk loads.
+    final LongHashSet allKeys = new LongHashSet(deferredOutHead.size() + deferredInHead.size());
+    for (final Long k : deferredOutHead.keySet())
+      allKeys.add(k);
+    for (final Long k : deferredInHead.keySet())
+      allKeys.add(k);
 
     // Sort by vertex key for page locality
-    final long[] sortedKeys = new long[allKeys.size()];
-    int ki = 0;
-    for (final long key : allKeys)
-      sortedKeys[ki++] = key;
+    final long[] sortedKeys = allKeys.toArray();
     Arrays.parallelSort(sortedKeys);
 
     beginTx();