tapdb: parallelize universe proof ingest

docs/release-notes/release-notes-0.8.2.md

@@ -45,11 +45,14 @@
 
 ## Performance Improvements
 
+- [PR#2183](https://github.com/lightninglabs/taproot-assets/pull/2183)
+  dramatically improves the performance of MS-SMT proof verification.
+
 - [PR#2184](https://github.com/lightninglabs/taproot-assets/pull/2184)
   dramatically improves the performance of universe federation proof push.
 
-- [PR#2183](https://github.com/lightninglabs/taproot-assets/pull/2183)
-  dramatically improves the performance of MS-SMT proof verification.
+- [PR#2194](https://github.com/lightninglabs/taproot-assets/pull/2194)
+  significantly improves concurrent universe proof ingest on Postgres.
 
 - [PR#2188](https://github.com/lightninglabs/taproot-assets/pull/2188)
   dramatically improves the performance of batched MS-SMT insertions.

mssmt/tree_prop_test.go

@@ -0,0 +1,99 @@
+package mssmt_test
+
+import (
+	"context"
+	"testing"
+
+	"github.com/lightninglabs/taproot-assets/mssmt"
+	"github.com/stretchr/testify/require"
+	"pgregory.net/rapid"
+)
+
+// testInsertLastWriteWins asserts that insertion is last-write-wins per
+// key: applying a sequence of inserts that reuses keys produces the
+// same tree as inserting only the final leaf observed for each key.
+// This is the invariant that allows coalescing consecutive updates to
+// the same key into a single insert of the latest value.
+func testInsertLastWriteWins(t *rapid.T) {
+	ctx := context.Background()
+
+	// Draw a small pool of keys so the insertion sequence is likely
+	// to hit the same key several times. Duplicate draws within the
+	// pool are harmless; they only shrink the effective pool.
+	numKeys := rapid.IntRange(1, 8).Draw(t, "num_keys")
+	keys := make([][hashSize]byte, numKeys)
+	for i := range keys {
+		keyBytes := rapid.SliceOfN(
+			rapid.Byte(), hashSize, hashSize,
+		).Draw(t, "key")
+		copy(keys[i][:], keyBytes)
+	}
+
+	// Draw the insertion sequence. Sums are bounded well below the
+	// point where the tree's uint64 sum overflow check could
+	// trigger.
+	numInserts := rapid.IntRange(1, 32).Draw(t, "num_inserts")
+	sequence := make([]treeLeaf, numInserts)
+	for i := range sequence {
+		keyIdx := rapid.IntRange(0, numKeys-1).Draw(t, "key_idx")
+		value := rapid.SliceOfN(rapid.Byte(), 1, 64).Draw(t, "value")
+		sum := rapid.Uint64Range(0, 1<<32).Draw(t, "sum")
+
+		sequence[i] = treeLeaf{
+			key:  keys[keyIdx],
+			leaf: mssmt.NewLeafNode(value, sum),
+		}
+	}
+
+	// Apply the full sequence in order.
+	full := mssmt.NewCompactedTree(mssmt.NewDefaultStore())
+	for _, item := range sequence {
+		_, err := full.Insert(ctx, item.key, item.leaf)
+		require.NoError(t, err)
+	}
+
+	// Reduce the sequence to the final leaf per key, keeping the
+	// order of each key's first occurrence, and apply only those.
+	finalLeaves := make(map[[hashSize]byte]*mssmt.LeafNode)
+	var keyOrder [][hashSize]byte
+	for _, item := range sequence {
+		if _, ok := finalLeaves[item.key]; !ok {
+			keyOrder = append(keyOrder, item.key)
+		}
+		finalLeaves[item.key] = item.leaf
+	}
+
+	coalesced := mssmt.NewCompactedTree(mssmt.NewDefaultStore())
+	for _, key := range keyOrder {
+		_, err := coalesced.Insert(ctx, key, finalLeaves[key])
+		require.NoError(t, err)
+	}
+
+	fullRoot, err := full.Root(ctx)
+	require.NoError(t, err)
+	coalescedRoot, err := coalesced.Root(ctx)
+	require.NoError(t, err)
+
+	require.True(
+		t, mssmt.IsEqualNode(fullRoot, coalescedRoot),
+		"full root %v != coalesced root %v", fullRoot, coalescedRoot,
+	)
+
+	// Each key's final leaf must carry a valid inclusion proof in
+	// the fully-inserted tree.
+	for _, key := range keyOrder {
+		proof, err := full.MerkleProof(ctx, key)
+		require.NoError(t, err)
+		require.True(t, mssmt.VerifyMerkleProof(
+			key, finalLeaves[key], proof, fullRoot,
+		))
+	}
+}
+
+// TestInsertLastWriteWins runs the last-write-wins insertion property
+// against the compacted tree.
+func TestInsertLastWriteWins(t *testing.T) {
+	t.Parallel()
+
+	rapid.Check(t, testInsertLastWriteWins)
+}

tapcfg/server.go

@@ -196,6 +196,15 @@ func genServerConfig(cfg *Config, cfgLogger btclog.Logger,
 		return nil, fmt.Errorf("create multiverse store: %w", err)
 	}
 
+	// The multiverse trees are derived from the universe roots; repair any
+	// entries that diverged, for example because the daemon stopped
+	// between a proof insert committing and its multiverse update being
+	// written.
+	err = multiverse.ReconcileMultiverse(context.Background())
+	if err != nil {
+		return nil, fmt.Errorf("reconcile multiverse: %w", err)
+	}
+
 	uniStatsDB := tapdb.NewTransactionExecutor(
 		db, func(tx *sql.Tx) tapdb.UniverseStatsStore {
 			return db.WithTx(tx)

tapdb/interfaces.go

@@ -322,13 +322,30 @@ type BaseDB struct {
 	*sqlc.Queries
 }
 
+// TxIsolationOverrider is an optional interface that TxOptions can implement
+// to override the default serializable isolation level a transaction is
+// started with. The override is only honored on the Postgres backend;
+// SQLite's driver only supports its default isolation.
+type TxIsolationOverrider interface {
+	// TxIsolation returns the isolation level to start the transaction
+	// with.
+	TxIsolation() sql.IsolationLevel
+}
+
 // BeginTx wraps the normal sql specific BeginTx method with the TxOptions
 // interface. This interface is then mapped to the concrete sql tx options
 // struct.
 func (s *BaseDB) BeginTx(ctx context.Context, opts TxOptions) (*sql.Tx, error) {
+	isolation := sql.LevelSerializable
+	if o, ok := opts.(TxIsolationOverrider); ok &&
+		s.Backend() == sqlc.BackendTypePostgres {
+
+		isolation = o.TxIsolation()
+	}
+
 	sqlOptions := sql.TxOptions{
 		ReadOnly:  opts.ReadOnly(),
-		Isolation: sql.LevelSerializable,
+		Isolation: isolation,
 	}
 	return s.DB.BeginTx(ctx, &sqlOptions)
 }