diff --git a/catchup/catchpointService.go b/catchup/catchpointService.go index 801901048b..80404d785e 100644 --- a/catchup/catchpointService.go +++ b/catchup/catchpointService.go @@ -674,7 +674,7 @@ func (cs *CatchpointCatchupService) fetchBlock(round basics.Round, retryCount ui return nil, time.Duration(0), psp, true, cs.abort(fmt.Errorf("fetchBlock: recurring non-HTTP peer was provided by the peer selector")) } fetcher := makeUniversalBlockFetcher(cs.log, cs.net, cs.config) - blk, _, downloadDuration, err = fetcher.fetchBlock(cs.ctx, round, httpPeer) + blk, _, _, downloadDuration, err = fetcher.fetchBlock(cs.ctx, round, httpPeer, false) if err != nil { if cs.ctx.Err() != nil { return nil, time.Duration(0), psp, true, cs.stopOrAbort() diff --git a/catchup/fetcher_test.go b/catchup/fetcher_test.go index 983de01475..176d36615a 100644 --- a/catchup/fetcher_test.go +++ b/catchup/fetcher_test.go @@ -31,6 +31,9 @@ import ( "github.com/algorand/go-algorand/components/mocks" "github.com/algorand/go-algorand/config" "github.com/algorand/go-algorand/crypto" + "github.com/algorand/go-algorand/crypto/merklearray" + "github.com/algorand/go-algorand/crypto/merklesignature" + cryptostateproof "github.com/algorand/go-algorand/crypto/stateproof" "github.com/algorand/go-algorand/data" "github.com/algorand/go-algorand/data/basics" "github.com/algorand/go-algorand/data/bookkeeping" @@ -38,18 +41,60 @@ import ( "github.com/algorand/go-algorand/logging" "github.com/algorand/go-algorand/network" "github.com/algorand/go-algorand/protocol" + "github.com/algorand/go-algorand/stateproof" ) -func buildTestLedger(t *testing.T, blk bookkeeping.Block) (ledger *data.Ledger, next basics.Round, b bookkeeping.Block, err error) { +const ( + testLedgerKeyValidRounds = 10000 +) + +type testLedgerStateProofData struct { + Params config.ConsensusParams + User basics.Address + Secrets *merklesignature.Secrets + TotalWeight basics.MicroAlgos + Participants basics.ParticipantsArray + Tree *merklearray.Tree + TemplateBlock bookkeeping.Block +} + +func buildTestLedger(t *testing.T, blk bookkeeping.Block) (ledger *data.Ledger, next basics.Round, b bookkeeping.Block, stateProofData *testLedgerStateProofData, err error) { var user basics.Address user[0] = 123 - proto := config.Consensus[protocol.ConsensusCurrentVersion] - genesis := make(map[basics.Address]basics.AccountData) - genesis[user] = basics.AccountData{ + ver := blk.CurrentProtocol + if ver == "" { + ver = protocol.ConsensusCurrentVersion + } + + proto := config.Consensus[ver] + + userData := basics.AccountData{ Status: basics.Offline, MicroAlgos: basics.MicroAlgos{Raw: proto.MinBalance * 2000000}, } + + if proto.StateProofInterval > 0 { + stateProofData = &testLedgerStateProofData{ + Params: proto, + User: user, + } + + stateProofData.Secrets, err = merklesignature.New(0, testLedgerKeyValidRounds, proto.StateProofInterval) + if err != nil { + t.Fatal("couldn't generate state proof keys", err) + return + } + + userData.StateProofID = stateProofData.Secrets.GetVerifier().Commitment + userData.VoteFirstValid = 0 + userData.VoteLastValid = testLedgerKeyValidRounds + userData.VoteKeyDilution = 1 + userData.Status = basics.Online + } + + genesis := make(map[basics.Address]basics.AccountData) + genesis[user] = userData genesis[sinkAddr] = basics.AccountData{ Status: basics.Offline, MicroAlgos: basics.MicroAlgos{Raw: proto.MinBalance * 2000000}, @@ -66,7 +111,7 @@ func buildTestLedger(t *testing.T, blk bookkeeping.Block) (ledger *data.Ledger, cfg := config.GetDefaultLocal() cfg.Archival = true ledger, err = data.LoadLedger( - log, t.Name(), inMem, protocol.ConsensusCurrentVersion, genBal, "", genHash, + log, t.Name(), inMem, ver, genBal, "", genHash, nil, cfg, ) if err != nil { @@ -99,7 +144,7 @@ func buildTestLedger(t *testing.T, blk bookkeeping.Block) (ledger *data.Ledger, b.RewardsLevel = prev.RewardsLevel b.BlockHeader.Round = next b.BlockHeader.GenesisHash = genHash - b.CurrentProtocol = protocol.ConsensusCurrentVersion + b.CurrentProtocol = ver txib, err := b.EncodeSignedTxn(signedtx, transactions.ApplyData{}) require.NoError(t, err) b.Payset = []transactions.SignedTxnInBlock{ @@ -107,15 +152,97 @@ func buildTestLedger(t *testing.T, blk bookkeeping.Block) (ledger *data.Ledger, } b.TxnCommitments, err = b.PaysetCommit() require.NoError(t, err) + + if proto.StateProofInterval > 0 { + var p basics.Participant + p.Weight = userData.MicroAlgos.ToUint64() + p.PK.KeyLifetime = merklesignature.KeyLifetimeDefault + p.PK.Commitment = userData.StateProofID + + stateProofData.Participants = append(stateProofData.Participants, p) + stateProofData.TotalWeight = userData.MicroAlgos + stateProofData.Tree, err = merklearray.BuildVectorCommitmentTree(stateProofData.Participants, crypto.HashFactory{HashType: cryptostateproof.HashType}) + if err != nil { + t.Fatal("couldn't build state proof voters tree", err) + return + } + + b.StateProofTracking = map[protocol.StateProofType]bookkeeping.StateProofTrackingData{ + protocol.StateProofBasic: { + StateProofVotersCommitment: stateProofData.Tree.Root(), + StateProofOnlineTotalWeight: stateProofData.TotalWeight, + StateProofNextRound: basics.Round(proto.StateProofInterval), + }, + } + } + require.NoError(t, ledger.AddBlock(b, agreement.Certificate{Round: next})) return } -func addBlocks(t *testing.T, ledger *data.Ledger, blk bookkeeping.Block, numBlocks int) { +func addBlocks(t *testing.T, ledger *data.Ledger, blk bookkeeping.Block, stateProofData *testLedgerStateProofData, numBlocks int) { var err error + origPayset := blk.Payset + nextStateProofTracking := blk.StateProofTracking + for i := 0; i < numBlocks; i++ { blk.BlockHeader.Round++ blk.BlockHeader.TimeStamp += int64(crypto.RandUint64() % 100 * 1000) + blk.Payset = origPayset + blk.StateProofTracking = nextStateProofTracking + + if stateProofData != nil && + (blk.BlockHeader.Round%basics.Round(stateProofData.Params.StateProofInterval)) == 0 && + blk.BlockHeader.Round > basics.Round(stateProofData.Params.StateProofInterval) { + proofrnd := blk.BlockHeader.Round.SubSaturate(basics.Round(stateProofData.Params.StateProofInterval)) + msg, err := stateproof.GenerateStateProofMessage(ledger, proofrnd) + require.NoError(t, err) + + provenWeight, overflowed := basics.Muldiv(stateProofData.TotalWeight.ToUint64(), uint64(stateProofData.Params.StateProofWeightThreshold), 1<<32) + require.False(t, overflowed) + + msgHash := msg.Hash() + prover, err := cryptostateproof.MakeProver(msgHash, + uint64(proofrnd), + provenWeight, + stateProofData.Participants, + stateProofData.Tree, + stateProofData.Params.StateProofStrengthTarget) + require.NoError(t, err) + + sig, err := stateProofData.Secrets.GetSigner(uint64(proofrnd)).SignBytes(msgHash[:]) + require.NoError(t, err) + + err = prover.Add(0, sig) + require.NoError(t, err) + + require.True(t, prover.Ready()) + sp, err := prover.CreateProof() + require.NoError(t, err) + + var stxn transactions.SignedTxn + stxn.Txn.Type = protocol.StateProofTx + stxn.Txn.Sender = transactions.StateProofSender + stxn.Txn.FirstValid = blk.BlockHeader.Round + stxn.Txn.LastValid = blk.BlockHeader.Round + stxn.Txn.GenesisHash = blk.BlockHeader.GenesisHash + stxn.Txn.StateProofTxnFields.StateProofType = protocol.StateProofBasic + stxn.Txn.StateProofTxnFields.StateProof = *sp + stxn.Txn.StateProofTxnFields.Message = msg + + txib, err := blk.EncodeSignedTxn(stxn, transactions.ApplyData{}) + require.NoError(t, err) + blk.Payset = make([]transactions.SignedTxnInBlock, len(origPayset)+1) + copy(blk.Payset[:], origPayset[:]) + blk.Payset[len(origPayset)] = txib + + sptracking := blk.StateProofTracking[protocol.StateProofBasic] + sptracking.StateProofNextRound = blk.BlockHeader.Round + nextStateProofTracking = map[protocol.StateProofType]bookkeeping.StateProofTrackingData{ + protocol.StateProofBasic: sptracking, + } + } + blk.TxnCommitments, err = blk.PaysetCommit() require.NoError(t, err) @@ -126,6 +253,10 @@ func addBlocks(t *testing.T, ledger *data.Ledger, blk bookkeeping.Block, numBloc require.NoError(t, err) require.Equal(t, blk.BlockHeader, hdr) } + + blk.Payset = origPayset + blk.StateProofTracking = nextStateProofTracking + stateProofData.TemplateBlock = blk } type basicRPCNode struct { @@ -143,6 +274,13 @@ func (b *basicRPCNode) RegisterHTTPHandler(path string, handler http.Handler) { b.rmux.Handle(path, handler) } +func (b *basicRPCNode) RegisterHTTPHandlerFunc(path string, handler func(response http.ResponseWriter, request *http.Request)) { + if b.rmux == nil { + b.rmux = mux.NewRouter() + } + b.rmux.HandleFunc(path, handler) +} + func (b *basicRPCNode) RegisterHandlers(dispatch []network.TaggedMessageHandler) { } diff --git a/catchup/pref_test.go b/catchup/pref_test.go index 377575f23a..3d580ef74b 100644 --- a/catchup/pref_test.go +++ b/catchup/pref_test.go @@ -66,7 +66,7 @@ func BenchmarkServiceFetchBlocks(b *testing.B) { require.NoError(b, err) // Make Service - syncer := MakeService(logging.TestingLog(b), defaultConfig, net, local, new(mockedAuthenticator), nil, nil) + syncer := MakeService(logging.TestingLog(b), defaultConfig, net, local, new(mockedAuthenticator), nil, nil, nil) b.StartTimer() syncer.Start() for w := 0; w < 1000; w++ { diff --git a/catchup/service.go b/catchup/service.go index bc23b3d736..a1545fdae4 100644 --- a/catchup/service.go +++ b/catchup/service.go @@ -25,16 +25,20 @@ import ( "sync/atomic" "time" + "github.com/algorand/go-deadlock" + "github.com/algorand/go-algorand/agreement" "github.com/algorand/go-algorand/config" "github.com/algorand/go-algorand/crypto" "github.com/algorand/go-algorand/data/basics" "github.com/algorand/go-algorand/data/bookkeeping" + "github.com/algorand/go-algorand/data/stateproofmsg" "github.com/algorand/go-algorand/ledger/ledgercore" "github.com/algorand/go-algorand/logging" "github.com/algorand/go-algorand/logging/telemetryspec" "github.com/algorand/go-algorand/network" "github.com/algorand/go-algorand/protocol" + "github.com/algorand/go-algorand/util/db" "github.com/algorand/go-algorand/util/execpool" ) @@ -66,6 +70,7 @@ type Ledger interface { Validate(ctx context.Context, blk bookkeeping.Block, executionPool execpool.BacklogPool) (*ledgercore.ValidatedBlock, error) AddValidatedBlock(vb ledgercore.ValidatedBlock, cert agreement.Certificate) error WaitMem(r basics.Round) chan struct{} + GetStateProofVerificationContext(basics.Round) (*ledgercore.StateProofVerificationContext, error) } // Service represents the catchup service. Once started and until it is stopped, it ensures that the ledger is up to date with network. @@ -105,6 +110,54 @@ type Service struct { // unsupportedRoundMonitor goroutine, after detecting // an unsupported block. onceUnsupportedRound sync.Once + + // stateproofs contains validated state proof messages for + // future rounds. The round is the LastAttestedRound of + // the state proof message. + // + // To help with garbage-collecting state proofs for rounds + // that are already in the ledger, stateproofmin tracks the + // lowest-numbered round in stateproofs. stateproofmin=0 + // indicates that no state proofs have been fetched. + // Similarly, stateproofmax tracks the most recent available + // state proof. + // + // stateproofdb stores these state proofs to ensure progress + // if algod gets restarted halfway through a long catchup. + // + // stateproofproto is the protocol version that corresponds + // to the consensus parameters for state proofs, either from + // a ledger round where we saw state proofs enabled, or from + // a special renaissance block configuration setting (below). + // + // stateproofproto is set if stateproofmin>0. + // + // stateproofmu protects the stateproof state from concurrent + // access. + // + // stateproofwait tracks channels for waiting on state proofs + // to be fetched. If stateproofwait is nil, there is no active + // stateProofFetcher that can be waited for. + stateproofs map[basics.Round]stateProofInfo + stateproofmin basics.Round + stateproofmax basics.Round + stateproofdb *db.Accessor + stateproofproto protocol.ConsensusVersion + stateproofmu deadlock.Mutex + stateproofwait map[basics.Round]chan struct{} + + // renaissance specifies the parameters for a renaissance + // block from which we can start validating state proofs, + // in lieu of validating the entire sequence of blocks + // starting from the genesis block. + renaissance *StateProofVerificationContext +} + +// stateProofInfo is a validated state proof message for some round, +// along with the consensus protocol for that round. +type stateProofInfo struct { + message stateproofmsg.Message + proto protocol.ConsensusVersion } // A BlockAuthenticator authenticates blocks given a certificate. @@ -120,7 +173,7 @@ type BlockAuthenticator interface { } // MakeService creates a catchup service instance from its constituent components -func MakeService(log logging.Logger, config config.Local, net network.GossipNode, ledger Ledger, auth BlockAuthenticator, unmatchedPendingCertificates <-chan PendingUnmatchedCertificate, blockValidationPool execpool.BacklogPool) (s *Service) { +func MakeService(log logging.Logger, config config.Local, net network.GossipNode, ledger Ledger, auth BlockAuthenticator, unmatchedPendingCertificates <-chan PendingUnmatchedCertificate, blockValidationPool execpool.BacklogPool, spCatchupDB *db.Accessor) (s *Service) { s = &Service{} s.cfg = config @@ -133,6 +186,16 @@ func MakeService(log logging.Logger, config config.Local, net network.GossipNode s.deadlineTimeout = agreement.DeadlineTimeout() s.blockValidationPool = blockValidationPool s.syncNow = make(chan struct{}, 1) + s.stateproofs = make(map[basics.Round]stateProofInfo) + + if spCatchupDB != nil { + s.stateproofdb = spCatchupDB + + err := s.initStateProofs() + if err != nil { + s.log.Warnf("catchup.initStateProofs(): %v", err) + } + } return s } @@ -209,13 +272,15 @@ func (s *Service) SynchronizingTime() time.Duration { // errLedgerAlreadyHasBlock is returned by innerFetch in case the local ledger already has the requested block. var errLedgerAlreadyHasBlock = errors.New("ledger already has block") -// function scope to make a bunch of defer statements better -func (s *Service) innerFetch(ctx context.Context, r basics.Round, peer network.Peer) (blk *bookkeeping.Block, cert *agreement.Certificate, ddur time.Duration, err error) { +// innerFetch retrieves a block with a certificate or state-proof-based +// light block header proof for round r from peer. proofOK specifies +// whether it's acceptable to fetch a proof instead of a certificate. +func (s *Service) innerFetch(ctx context.Context, r basics.Round, peer network.Peer, proofOK bool) (blk *bookkeeping.Block, cert *agreement.Certificate, proof []byte, ddur time.Duration, err error) { ledgerWaitCh := s.ledger.WaitMem(r) select { case <-ledgerWaitCh: // if our ledger already have this block, no need to attempt to fetch it. - return nil, nil, time.Duration(0), errLedgerAlreadyHasBlock + return nil, nil, nil, time.Duration(0), errLedgerAlreadyHasBlock default: } @@ -229,7 +294,7 @@ func (s *Service) innerFetch(ctx context.Context, r basics.Round, peer network.P cf() } }() - blk, cert, ddur, err = fetcher.fetchBlock(ctx, r, peer) + blk, cert, proof, ddur, err = fetcher.fetchBlock(ctx, r, peer, proofOK) // check to see if we aborted due to ledger. if err != nil { select { @@ -291,8 +356,23 @@ func (s *Service) fetchAndWrite(ctx context.Context, r basics.Round, prevFetchCo } peer := psp.Peer + // Wait for a state proof to become available for this block. + // If no state proofs are available, this will return right away. + select { + case <-ctx.Done(): + s.log.Debugf("fetchAndWrite(%v): Aborted", r) + return false + case <-s.stateProofWait(r): + } + + spinfo := s.getStateProof(r) + // Try to fetch, timing out after retryInterval - block, cert, blockDownloadDuration, err := s.innerFetch(ctx, r, peer) + proofOK := true + if spinfo == nil || !config.Consensus[spinfo.proto].StateProofBlockHashInLightHeader { + proofOK = false + } + block, cert, proof, blockDownloadDuration, err := s.innerFetch(ctx, r, peer, proofOK) if err != nil { if err == errLedgerAlreadyHasBlock { @@ -314,11 +394,11 @@ func (s *Service) fetchAndWrite(ctx context.Context, r basics.Round, prevFetchCo case <-lookbackComplete: } continue // retry the fetch - } else if block == nil || cert == nil { + } else if block == nil { // someone already wrote the block to the ledger, we should stop syncing return false } - s.log.Debugf("fetchAndWrite(%v): Got block and cert contents: %v %v", r, block, cert) + s.log.Debugf("fetchAndWrite(%v): Got block and cert/proof contents: %v %v %v", r, block, cert, proof) // Check that the block's contents match the block header (necessary with an untrusted block because b.Hash() only hashes the header) if s.cfg.CatchupVerifyPaysetHash() { @@ -335,21 +415,30 @@ func (s *Service) fetchAndWrite(ctx context.Context, r basics.Round, prevFetchCo } } - // make sure that we have the lookBack block that's required for authenticating this block - select { - case <-ctx.Done(): - s.log.Debugf("fetchAndWrite(%v): Aborted while waiting for lookback block to ledger", r) - return false - case <-lookbackComplete: - } - - if s.cfg.CatchupVerifyCertificate() { - err = s.auth.Authenticate(block, cert) + if spinfo != nil && len(proof) > 0 { + err = verifyBlockStateProof(r, &spinfo.message, block, proof) if err != nil { - s.log.Warnf("fetchAndWrite(%v): cert did not authenticate block (attempt %d): %v", r, i, err) + s.log.Warnf("fetchAndWrite(%v): proof did not authenticate block (attempt %d): %v", r, i, err) peerSelector.rankPeer(psp, peerRankInvalidDownload) continue // retry the fetch } + } else { + // make sure that we have the lookBack block that's required for authenticating this block + select { + case <-ctx.Done(): + s.log.Debugf("fetchAndWrite(%v): Aborted while waiting for lookback block to ledger", r) + return false + case <-lookbackComplete: + } + + if s.cfg.CatchupVerifyCertificate() { + err = s.auth.Authenticate(block, cert) + if err != nil { + s.log.Warnf("fetchAndWrite(%v): cert did not authenticate block (attempt %d): %v", r, i, err) + peerSelector.rankPeer(psp, peerRankInvalidDownload) + continue // retry the fetch + } + } } peerRank := peerSelector.peerDownloadDurationToRank(psp, blockDownloadDuration) @@ -453,6 +542,10 @@ func (s *Service) pipelinedFetch(seedLookback uint64) { ctx, cancelCtx := context.WithCancel(s.ctx) defer cancelCtx() + // Start the state proof fetcher, which will enable us to validate + // blocks using state proofs if available. + s.startStateProofFetcher(ctx) + // firstRound is the first round we're waiting to fetch. firstRound := s.ledger.NextRound() @@ -680,7 +773,7 @@ func (s *Service) fetchRound(cert agreement.Certificate, verifier *agreement.Asy peer := psp.Peer // Ask the fetcher to get the block somehow - block, fetchedCert, _, err := s.innerFetch(s.ctx, cert.Round, peer) + block, fetchedCert, _, _, err := s.innerFetch(s.ctx, cert.Round, peer, false) if err != nil { select { diff --git a/catchup/service_test.go b/catchup/service_test.go index 406f5ef819..3264ba8fea 100644 --- a/catchup/service_test.go +++ b/catchup/service_test.go @@ -137,12 +137,12 @@ func TestServiceFetchBlocksSameRange(t *testing.T) { local := new(mockedLedger) local.blocks = append(local.blocks, bookkeeping.Block{}) - remote, _, blk, err := buildTestLedger(t, bookkeeping.Block{}) + remote, _, blk, spdata, err := buildTestLedger(t, bookkeeping.Block{}) if err != nil { t.Fatal(err) return } - addBlocks(t, remote, blk, 10) + addBlocks(t, remote, blk, spdata, 10) // Create a network and block service blockServiceConfig := config.GetDefaultLocal() @@ -157,7 +157,7 @@ func TestServiceFetchBlocksSameRange(t *testing.T) { net.addPeer(rootURL) // Make Service - syncer := MakeService(logging.Base(), defaultConfig, net, local, &mockedAuthenticator{errorRound: -1}, nil, nil) + syncer := MakeService(logging.Base(), defaultConfig, net, local, &mockedAuthenticator{errorRound: -1}, nil, nil, nil) syncer.testStart() syncer.sync() @@ -188,12 +188,12 @@ func TestSyncRound(t *testing.T) { local := new(mockedLedger) local.blocks = append(local.blocks, bookkeeping.Block{}) - remote, _, blk, err := buildTestLedger(t, bookkeeping.Block{}) + remote, _, blk, spdata, err := buildTestLedger(t, bookkeeping.Block{}) if err != nil { t.Fatal(err) return } - addBlocks(t, remote, blk, 10) + addBlocks(t, remote, blk, spdata, 10) // Create a network and block service blockServiceConfig := config.GetDefaultLocal() @@ -213,7 +213,7 @@ func TestSyncRound(t *testing.T) { // Make Service localCfg := config.GetDefaultLocal() - s := MakeService(logging.Base(), localCfg, net, local, auth, nil, nil) + s := MakeService(logging.Base(), localCfg, net, local, auth, nil, nil, nil) s.log = &periodicSyncLogger{Logger: logging.Base()} s.deadlineTimeout = 2 * time.Second @@ -278,12 +278,12 @@ func TestPeriodicSync(t *testing.T) { local := new(mockedLedger) local.blocks = append(local.blocks, bookkeeping.Block{}) - remote, _, blk, err := buildTestLedger(t, bookkeeping.Block{}) + remote, _, blk, spdata, err := buildTestLedger(t, bookkeeping.Block{}) if err != nil { t.Fatal(err) return } - addBlocks(t, remote, blk, 10) + addBlocks(t, remote, blk, spdata, 10) // Create a network and block service blockServiceConfig := config.GetDefaultLocal() @@ -302,7 +302,7 @@ func TestPeriodicSync(t *testing.T) { require.True(t, 0 == initialLocalRound) // Make Service - s := MakeService(logging.Base(), defaultConfig, net, local, auth, nil, nil) + s := MakeService(logging.Base(), defaultConfig, net, local, auth, nil, nil, nil) s.log = &periodicSyncLogger{Logger: logging.Base()} s.deadlineTimeout = 2 * time.Second @@ -344,12 +344,12 @@ func TestServiceFetchBlocksOneBlock(t *testing.T) { local.blocks = append(local.blocks, bookkeeping.Block{}) lastRoundAtStart := local.LastRound() - remote, _, blk, err := buildTestLedger(t, bookkeeping.Block{}) + remote, _, blk, spdata, err := buildTestLedger(t, bookkeeping.Block{}) if err != nil { t.Fatal(err) return } - addBlocks(t, remote, blk, numBlocks-1) + addBlocks(t, remote, blk, spdata, numBlocks-1) // Create a network and block service blockServiceConfig := config.GetDefaultLocal() @@ -364,7 +364,7 @@ func TestServiceFetchBlocksOneBlock(t *testing.T) { net.addPeer(rootURL) // Make Service - s := MakeService(logging.Base(), defaultConfig, net, local, &mockedAuthenticator{errorRound: -1}, nil, nil) + s := MakeService(logging.Base(), defaultConfig, net, local, &mockedAuthenticator{errorRound: -1}, nil, nil, nil) // Get last round @@ -378,9 +378,9 @@ func TestServiceFetchBlocksOneBlock(t *testing.T) { require.Equal(t, lastRoundAtStart+basics.Round(numBlocks), local.LastRound()) // Get the same block we wrote - block, _, _, err := makeUniversalBlockFetcher(logging.Base(), + block, _, _, _, err := makeUniversalBlockFetcher(logging.Base(), net, - defaultConfig).fetchBlock(context.Background(), lastRoundAtStart+1, net.peers[0]) + defaultConfig).fetchBlock(context.Background(), lastRoundAtStart+1, net.peers[0], false) require.NoError(t, err) @@ -408,12 +408,12 @@ func TestAbruptWrites(t *testing.T) { lastRound := local.LastRound() - remote, _, blk, err := buildTestLedger(t, bookkeeping.Block{}) + remote, _, blk, spdata, err := buildTestLedger(t, bookkeeping.Block{}) if err != nil { t.Fatal(err) return } - addBlocks(t, remote, blk, numberOfBlocks-1) + addBlocks(t, remote, blk, spdata, numberOfBlocks-1) // Create a network and block service blockServiceConfig := config.GetDefaultLocal() @@ -428,7 +428,7 @@ func TestAbruptWrites(t *testing.T) { net.addPeer(rootURL) // Make Service - s := MakeService(logging.Base(), defaultConfig, net, local, &mockedAuthenticator{errorRound: -1}, nil, nil) + s := MakeService(logging.Base(), defaultConfig, net, local, &mockedAuthenticator{errorRound: -1}, nil, nil, nil) var wg sync.WaitGroup wg.Add(1) @@ -466,12 +466,12 @@ func TestServiceFetchBlocksMultiBlocks(t *testing.T) { lastRoundAtStart := local.LastRound() - remote, _, blk, err := buildTestLedger(t, bookkeeping.Block{}) + remote, _, blk, spdata, err := buildTestLedger(t, bookkeeping.Block{}) if err != nil { t.Fatal(err) return } - addBlocks(t, remote, blk, int(numberOfBlocks)-1) + addBlocks(t, remote, blk, spdata, int(numberOfBlocks)-1) // Create a network and block service blockServiceConfig := config.GetDefaultLocal() @@ -486,7 +486,7 @@ func TestServiceFetchBlocksMultiBlocks(t *testing.T) { net.addPeer(rootURL) // Make Service - syncer := MakeService(logging.Base(), defaultConfig, net, local, &mockedAuthenticator{errorRound: -1}, nil, nil) + syncer := MakeService(logging.Base(), defaultConfig, net, local, &mockedAuthenticator{errorRound: -1}, nil, nil, nil) fetcher := makeUniversalBlockFetcher(logging.Base(), net, defaultConfig) // Start the service ( dummy ) @@ -500,7 +500,7 @@ func TestServiceFetchBlocksMultiBlocks(t *testing.T) { for i := basics.Round(1); i <= numberOfBlocks; i++ { // Get the same block we wrote - blk, _, _, err2 := fetcher.fetchBlock(context.Background(), i, net.GetPeers()[0]) + blk, _, _, _, err2 := fetcher.fetchBlock(context.Background(), i, net.GetPeers()[0], false) require.NoError(t, err2) // Check we wrote the correct block @@ -521,12 +521,12 @@ func TestServiceFetchBlocksMalformed(t *testing.T) { lastRoundAtStart := local.LastRound() - remote, _, blk, err := buildTestLedger(t, bookkeeping.Block{}) + remote, _, blk, spdata, err := buildTestLedger(t, bookkeeping.Block{}) if err != nil { t.Fatal(err) return } - addBlocks(t, remote, blk, numBlocks-1) + addBlocks(t, remote, blk, spdata, numBlocks-1) // Create a network and block service blockServiceConfig := config.GetDefaultLocal() @@ -541,7 +541,7 @@ func TestServiceFetchBlocksMalformed(t *testing.T) { net.addPeer(rootURL) // Make Service - s := MakeService(logging.Base(), defaultConfig, net, local, &mockedAuthenticator{errorRound: int(lastRoundAtStart + 1)}, nil, nil) + s := MakeService(logging.Base(), defaultConfig, net, local, &mockedAuthenticator{errorRound: int(lastRoundAtStart + 1)}, nil, nil, nil) s.log = &periodicSyncLogger{Logger: logging.Base()} // Start the service ( dummy ) @@ -670,7 +670,7 @@ func helperTestOnSwitchToUnSupportedProtocol( config.CatchupParallelBlocks = 2 block1 := mRemote.blocks[1] - remote, _, blk, err := buildTestLedger(t, block1) + remote, _, blk, spdata, err := buildTestLedger(t, block1) if err != nil { t.Fatal(err) return local, remote @@ -679,7 +679,7 @@ func helperTestOnSwitchToUnSupportedProtocol( blk.NextProtocolSwitchOn = mRemote.blocks[i+1].NextProtocolSwitchOn blk.NextProtocol = mRemote.blocks[i+1].NextProtocol // Adds blk.BlockHeader.Round + 1 - addBlocks(t, remote, blk, 1) + addBlocks(t, remote, blk, spdata, 1) blk.BlockHeader.Round++ } @@ -695,7 +695,7 @@ func helperTestOnSwitchToUnSupportedProtocol( net.addPeer(rootURL) // Make Service - s := MakeService(logging.Base(), config, net, local, &mockedAuthenticator{errorRound: -1}, nil, nil) + s := MakeService(logging.Base(), config, net, local, &mockedAuthenticator{errorRound: -1}, nil, nil, nil) s.deadlineTimeout = 2 * time.Second s.Start() defer s.Stop() @@ -710,6 +710,7 @@ type mockedLedger struct { mu deadlock.Mutex blocks []bookkeeping.Block chans map[basics.Round]chan struct{} + certs map[basics.Round]agreement.Certificate } func (m *mockedLedger) NextRound() basics.Round { @@ -746,6 +747,12 @@ func (m *mockedLedger) AddBlock(blk bookkeeping.Block, cert agreement.Certificat delete(m.chans, r) } } + + if m.certs == nil { + m.certs = make(map[basics.Round]agreement.Certificate) + } + m.certs[blk.Round()] = cert + return nil } @@ -797,6 +804,15 @@ func (m *mockedLedger) Block(r basics.Round) (bookkeeping.Block, error) { return m.blocks[r], nil } +func (m *mockedLedger) BlockCert(r basics.Round) (bookkeeping.Block, agreement.Certificate, error) { + m.mu.Lock() + defer m.mu.Unlock() + if r > m.lastRound() { + return bookkeeping.Block{}, agreement.Certificate{}, errors.New("mockedLedger.Block: round too high") + } + return m.blocks[r], m.certs[r], nil +} + func (m *mockedLedger) BlockHdr(r basics.Round) (bookkeeping.BlockHeader, error) { blk, err := m.Block(r) return blk.BlockHeader, err @@ -830,6 +846,34 @@ func (m *mockedLedger) IsWritingCatchpointDataFile() bool { return false } +func (m *mockedLedger) GetStateProofVerificationContext(r basics.Round) (*ledgercore.StateProofVerificationContext, error) { + latest := m.LastRound() + latestHdr, err := m.BlockHdr(latest) + if err != nil { + return nil, err + } + + interval := basics.Round(config.Consensus[latestHdr.CurrentProtocol].StateProofInterval) + if interval == 0 { + return nil, errors.New("state proofs not supported") + } + + lastAttested := r.RoundUpToMultipleOf(interval) + votersRnd := lastAttested - interval + votersHdr, err := m.BlockHdr(votersRnd) + if err != nil { + return nil, err + } + + vc := &ledgercore.StateProofVerificationContext{ + LastAttestedRound: lastAttested, + VotersCommitment: votersHdr.StateProofTracking[protocol.StateProofBasic].StateProofVotersCommitment, + OnlineTotalWeight: votersHdr.StateProofTracking[protocol.StateProofBasic].StateProofOnlineTotalWeight, + Version: votersHdr.CurrentProtocol, + } + return vc, nil +} + func testingenvWithUpgrade( t testing.TB, numBlocks, @@ -885,13 +929,13 @@ func TestCatchupUnmatchedCertificate(t *testing.T) { local.blocks = append(local.blocks, bookkeeping.Block{}) lastRoundAtStart := local.LastRound() - remote, _, blk, err := buildTestLedger(t, bookkeeping.Block{}) + remote, _, blk, spdata, err := buildTestLedger(t, bookkeeping.Block{}) if err != nil { t.Fatal(err) return } defer remote.Close() - addBlocks(t, remote, blk, numBlocks-1) + addBlocks(t, remote, blk, spdata, numBlocks-1) // Create a network and block service blockServiceConfig := config.GetDefaultLocal() @@ -906,7 +950,7 @@ func TestCatchupUnmatchedCertificate(t *testing.T) { net.addPeer(rootURL) // Make Service - s := MakeService(logging.Base(), defaultConfig, net, local, &mockedAuthenticator{errorRound: int(lastRoundAtStart + 1)}, nil, nil) + s := MakeService(logging.Base(), defaultConfig, net, local, &mockedAuthenticator{errorRound: int(lastRoundAtStart + 1)}, nil, nil, nil) s.testStart() for roundNumber := 2; roundNumber < 10; roundNumber += 3 { pc := &PendingUnmatchedCertificate{ @@ -931,7 +975,7 @@ func TestCreatePeerSelector(t *testing.T) { cfg.EnableCatchupFromArchiveServers = true cfg.NetAddress = "someAddress" - s := MakeService(logging.Base(), cfg, &httpTestPeerSource{}, new(mockedLedger), &mockedAuthenticator{errorRound: int(0 + 1)}, nil, nil) + s := MakeService(logging.Base(), cfg, &httpTestPeerSource{}, new(mockedLedger), &mockedAuthenticator{errorRound: int(0 + 1)}, nil, nil, nil) ps := createPeerSelector(s.net, s.cfg, true) require.Equal(t, 5, len(ps.peerClasses)) require.Equal(t, peerRankInitialFirstPriority, ps.peerClasses[0].initialRank) @@ -949,7 +993,7 @@ func TestCreatePeerSelector(t *testing.T) { // cfg.EnableCatchupFromArchiveServers = true; cfg.NetAddress == ""; pipelineFetch = true; cfg.EnableCatchupFromArchiveServers = true cfg.NetAddress = "" - s = MakeService(logging.Base(), cfg, &httpTestPeerSource{}, new(mockedLedger), &mockedAuthenticator{errorRound: int(0 + 1)}, nil, nil) + s = MakeService(logging.Base(), cfg, &httpTestPeerSource{}, new(mockedLedger), &mockedAuthenticator{errorRound: int(0 + 1)}, nil, nil, nil) ps = createPeerSelector(s.net, s.cfg, true) require.Equal(t, 4, len(ps.peerClasses)) require.Equal(t, peerRankInitialFirstPriority, ps.peerClasses[0].initialRank) @@ -965,7 +1009,7 @@ func TestCreatePeerSelector(t *testing.T) { // cfg.EnableCatchupFromArchiveServers = true; cfg.NetAddress != ""; pipelineFetch = false cfg.EnableCatchupFromArchiveServers = true cfg.NetAddress = "someAddress" - s = MakeService(logging.Base(), cfg, &httpTestPeerSource{}, new(mockedLedger), &mockedAuthenticator{errorRound: int(0 + 1)}, nil, nil) + s = MakeService(logging.Base(), cfg, &httpTestPeerSource{}, new(mockedLedger), &mockedAuthenticator{errorRound: int(0 + 1)}, nil, nil, nil) ps = createPeerSelector(s.net, s.cfg, false) require.Equal(t, 5, len(ps.peerClasses)) @@ -984,7 +1028,7 @@ func TestCreatePeerSelector(t *testing.T) { // cfg.EnableCatchupFromArchiveServers = true; cfg.NetAddress == ""; pipelineFetch = false cfg.EnableCatchupFromArchiveServers = true cfg.NetAddress = "" - s = MakeService(logging.Base(), cfg, &httpTestPeerSource{}, new(mockedLedger), &mockedAuthenticator{errorRound: int(0 + 1)}, nil, nil) + s = MakeService(logging.Base(), cfg, &httpTestPeerSource{}, new(mockedLedger), &mockedAuthenticator{errorRound: int(0 + 1)}, nil, nil, nil) ps = createPeerSelector(s.net, s.cfg, false) require.Equal(t, 4, len(ps.peerClasses)) @@ -1001,7 +1045,7 @@ func TestCreatePeerSelector(t *testing.T) { // cfg.EnableCatchupFromArchiveServers = false; cfg.NetAddress != ""; pipelineFetch = true cfg.EnableCatchupFromArchiveServers = false cfg.NetAddress = "someAddress" - s = MakeService(logging.Base(), cfg, &httpTestPeerSource{}, new(mockedLedger), &mockedAuthenticator{errorRound: int(0 + 1)}, nil, nil) + s = MakeService(logging.Base(), cfg, &httpTestPeerSource{}, new(mockedLedger), &mockedAuthenticator{errorRound: int(0 + 1)}, nil, nil, nil) ps = createPeerSelector(s.net, s.cfg, true) require.Equal(t, 4, len(ps.peerClasses)) @@ -1018,7 +1062,7 @@ func TestCreatePeerSelector(t *testing.T) { // cfg.EnableCatchupFromArchiveServers = false; cfg.NetAddress == ""; pipelineFetch = true cfg.EnableCatchupFromArchiveServers = false cfg.NetAddress = "" - s = MakeService(logging.Base(), cfg, &httpTestPeerSource{}, new(mockedLedger), &mockedAuthenticator{errorRound: int(0 + 1)}, nil, nil) + s = MakeService(logging.Base(), cfg, &httpTestPeerSource{}, new(mockedLedger), &mockedAuthenticator{errorRound: int(0 + 1)}, nil, nil, nil) ps = createPeerSelector(s.net, s.cfg, true) require.Equal(t, 3, len(ps.peerClasses)) @@ -1033,7 +1077,7 @@ func TestCreatePeerSelector(t *testing.T) { // cfg.EnableCatchupFromArchiveServers = false; cfg.NetAddress != ""; pipelineFetch = false cfg.EnableCatchupFromArchiveServers = false cfg.NetAddress = "someAddress" - s = MakeService(logging.Base(), cfg, &httpTestPeerSource{}, new(mockedLedger), &mockedAuthenticator{errorRound: int(0 + 1)}, nil, nil) + s = MakeService(logging.Base(), cfg, &httpTestPeerSource{}, new(mockedLedger), &mockedAuthenticator{errorRound: int(0 + 1)}, nil, nil, nil) ps = createPeerSelector(s.net, s.cfg, false) require.Equal(t, 4, len(ps.peerClasses)) @@ -1050,7 +1094,7 @@ func TestCreatePeerSelector(t *testing.T) { // cfg.EnableCatchupFromArchiveServers = false; cfg.NetAddress == ""; pipelineFetch = false cfg.EnableCatchupFromArchiveServers = false cfg.NetAddress = "" - s = MakeService(logging.Base(), cfg, &httpTestPeerSource{}, new(mockedLedger), &mockedAuthenticator{errorRound: int(0 + 1)}, nil, nil) + s = MakeService(logging.Base(), cfg, &httpTestPeerSource{}, new(mockedLedger), &mockedAuthenticator{errorRound: int(0 + 1)}, nil, nil, nil) ps = createPeerSelector(s.net, s.cfg, false) require.Equal(t, 3, len(ps.peerClasses)) @@ -1069,7 +1113,7 @@ func TestServiceStartStop(t *testing.T) { cfg := defaultConfig ledger := new(mockedLedger) ledger.blocks = append(ledger.blocks, bookkeeping.Block{}) - s := MakeService(logging.Base(), cfg, &httpTestPeerSource{}, ledger, &mockedAuthenticator{errorRound: int(0 + 1)}, nil, nil) + s := MakeService(logging.Base(), cfg, &httpTestPeerSource{}, ledger, &mockedAuthenticator{errorRound: int(0 + 1)}, nil, nil, nil) s.Start() s.Stop() @@ -1082,7 +1126,7 @@ func TestSynchronizingTime(t *testing.T) { cfg := defaultConfig ledger := new(mockedLedger) ledger.blocks = append(ledger.blocks, bookkeeping.Block{}) - s := MakeService(logging.Base(), cfg, &httpTestPeerSource{}, ledger, &mockedAuthenticator{errorRound: int(0 + 1)}, nil, nil) + s := MakeService(logging.Base(), cfg, &httpTestPeerSource{}, ledger, &mockedAuthenticator{errorRound: int(0 + 1)}, nil, nil, nil) require.Equal(t, time.Duration(0), s.SynchronizingTime()) atomic.StoreInt64(&s.syncStartNS, 1000000) diff --git a/catchup/stateproof.go b/catchup/stateproof.go new file mode 100644 index 0000000000..e819470cd2 --- /dev/null +++ b/catchup/stateproof.go @@ -0,0 +1,483 @@ +// Copyright (C) 2019-2023 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see . + +package catchup + +import ( + "context" + "database/sql" + "encoding/base64" + "fmt" + + "github.com/algorand/go-algorand/config" + "github.com/algorand/go-algorand/crypto" + "github.com/algorand/go-algorand/crypto/merklearray" + "github.com/algorand/go-algorand/crypto/stateproof" + "github.com/algorand/go-algorand/data/basics" + "github.com/algorand/go-algorand/data/bookkeeping" + "github.com/algorand/go-algorand/data/stateproofmsg" + "github.com/algorand/go-algorand/protocol" + "github.com/algorand/go-algorand/util/db" +) + +// This file implements state-proof-based validation of new blocks, +// for catching up the state of a node with the rest of the network. + +// StateProofVerificationContext specifies the parameters needed to +// verify a state proof for the catchup code. +type StateProofVerificationContext struct { + // LastRound is the LastAttestedRound in the state proof message + // that we expect to verify with these parameters. + LastRound basics.Round + + // LnProvenWeight is passed to stateproof.MkVerifierWithLnProvenWeight. + LnProvenWeight uint64 + + // VotersCommitment is passed to stateproof.MkVerifierWithLnProvenWeight. + VotersCommitment crypto.GenericDigest + + // Proto specifies the protocol in which state proofs were enabled, + // used to determine StateProofStrengthTarget and StateProofInterval. + Proto protocol.ConsensusVersion +} + +func spSchemaUpgrade0(_ context.Context, tx *sql.Tx, _ bool) error { + const createProofsTable = `CREATE TABLE IF NOT EXISTS proofs ( + lastrnd integer, + proto text, + msg blob, + UNIQUE (lastrnd))` + + _, err := tx.Exec(createProofsTable) + return err +} + +func (s *Service) initStateProofs() error { + s.stateproofmu.Lock() + defer s.stateproofmu.Unlock() + + if s.stateproofdb == nil { + return nil + } + + migrations := []db.Migration{ + spSchemaUpgrade0, + } + + err := db.Initialize(*s.stateproofdb, migrations) + if err != nil { + return err + } + + stateproofs := make(map[basics.Round]stateProofInfo) + var stateproofmin basics.Round + var stateproofmax basics.Round + var stateproofproto protocol.ConsensusVersion + + err = s.stateproofdb.Atomic(func(ctx context.Context, tx *sql.Tx) error { + rows, err := tx.Query("SELECT proto, msg FROM proofs ORDER BY lastrnd") + if err != nil { + return err + } + + defer rows.Close() + for rows.Next() { + var proto protocol.ConsensusVersion + var msgbuf []byte + err := rows.Scan(&proto, &msgbuf) + if err != nil { + s.log.Warnf("initStateProofs: cannot scan proof from db: %v", err) + continue + } + + var msg stateproofmsg.Message + err = protocol.Decode(msgbuf, &msg) + if err != nil { + s.log.Warnf("initStateProofs: cannot decode proof from db: %v", err) + continue + } + + stateproofs[msg.LastAttestedRound] = stateProofInfo{ + message: msg, + proto: proto, + } + stateproofmax = msg.LastAttestedRound + if stateproofmin == 0 { + stateproofmin = msg.LastAttestedRound + stateproofproto = proto + } + } + return nil + }) + if err != nil { + return err + } + + s.stateproofs = stateproofs + s.stateproofmin = stateproofmin + s.stateproofmax = stateproofmax + s.stateproofproto = stateproofproto + + return nil +} + +// addStateProof adds a verified state proof message. +func (s *Service) addStateProof(msg stateproofmsg.Message, proto protocol.ConsensusVersion) { + s.stateproofmu.Lock() + defer s.stateproofmu.Unlock() + + if s.stateproofdb != nil { + err := s.stateproofdb.Atomic(func(ctx context.Context, tx *sql.Tx) error { + _, err := tx.Exec("INSERT INTO proofs (lastrnd, proto, msg) VALUES (?, ?, ?)", + msg.LastAttestedRound, proto, protocol.Encode(&msg)) + return err + }) + if err != nil { + s.log.Warnf("addStateProof: database error: %v", err) + } + } + + if s.stateproofmin == 0 { + s.stateproofmin = msg.LastAttestedRound + s.stateproofproto = proto + } + if msg.LastAttestedRound > s.stateproofmax { + s.stateproofmax = msg.LastAttestedRound + } + s.stateproofs[msg.LastAttestedRound] = stateProofInfo{ + message: msg, + proto: proto, + } + + for r := msg.FirstAttestedRound; r < msg.LastAttestedRound; r++ { + ch, ok := s.stateproofwait[r] + if ok { + close(ch) + delete(s.stateproofwait, r) + } + } +} + +// cleanupStateProofs removes state proofs that are for the latest +// round or earlier. +func (s *Service) cleanupStateProofs(latest basics.Round) { + s.stateproofmu.Lock() + defer s.stateproofmu.Unlock() + + if s.stateproofmin == 0 { + return + } + + if s.stateproofdb != nil { + err := s.stateproofdb.Atomic(func(ctx context.Context, tx *sql.Tx) error { + _, err := tx.Exec("DELETE FROM proofs WHERE lastrnd<=?", latest) + return err + }) + if err != nil { + s.log.Warnf("cleanupStateProofs: database error: %v", err) + } + } + + for s.stateproofmin <= latest { + delete(s.stateproofs, s.stateproofmin) + s.stateproofmin += basics.Round(config.Consensus[s.stateproofproto].StateProofInterval) + } +} + +// nextStateProofVerifier() returns the latest state proof verification +// context that we have access to. This might be based on the latest block +// in the ledger, or based on the latest state proof (beyond the end of the +// ledger) that we have, or based on well-known "renaissance block" values. +// +// The return value might be nil if no verification context is available. +func (s *Service) nextStateProofVerifier() *StateProofVerificationContext { + s.stateproofmu.Lock() + defer s.stateproofmu.Unlock() + + // As a baseline, use the renaissance verification context (if present). + res := s.renaissance + + // Check if we have a more recent verified state proof in memory. + lastProof, ok := s.stateproofs[s.stateproofmax] + if ok && (res == nil || lastProof.message.LastAttestedRound >= res.LastRound) { + res = &StateProofVerificationContext{ + LastRound: lastProof.message.LastAttestedRound + basics.Round(config.Consensus[lastProof.proto].StateProofInterval), + LnProvenWeight: lastProof.message.LnProvenWeight, + VotersCommitment: lastProof.message.VotersCommitment, + Proto: s.stateproofproto, + } + } + + // Check if the ledger has a more recent state proof verification context. + latest := s.ledger.LastRound() + + // If we don't know state proof parameters yet, check the ledger. + proto := s.stateproofproto + params, paramsOk := config.Consensus[proto] + if !paramsOk { + hdr, err := s.ledger.BlockHdr(latest) + if err != nil { + s.log.Warnf("nextStateProofVerifier: BlockHdr(%d): %v", latest, err) + } else { + proto = hdr.CurrentProtocol + params, paramsOk = config.Consensus[proto] + } + } + + if !paramsOk || params.StateProofInterval == 0 { + // Ledger's latest block does not support state proof. + // Return whatever verification context we've found so far. + return res + } + + // The next state proof verification context we should expect from + // the ledger is for StateProofInterval in the future from the most + // recent multiple of StateProofInterval. + nextLastRound := latest.RoundDownToMultipleOf(basics.Round(params.StateProofInterval)) + basics.Round(params.StateProofInterval) + if res != nil && nextLastRound <= res.LastRound { + // We already have a verification context that's no older. + return res + } + + vctx, err := s.ledger.GetStateProofVerificationContext(nextLastRound) + if err != nil { + s.log.Warnf("nextStateProofVerifier: GetStateProofVerificationContext(%d): %v", nextLastRound, err) + return res + } + + provenWeight, overflowed := basics.Muldiv(vctx.OnlineTotalWeight.ToUint64(), uint64(params.StateProofWeightThreshold), 1<<32) + if overflowed { + s.log.Warnf("nextStateProofVerifier: overflow computing provenWeight[%d]: %d * %d / (1<<32)", + nextLastRound, vctx.OnlineTotalWeight.ToUint64(), params.StateProofWeightThreshold) + return res + } + + lnProvenWt, err := stateproof.LnIntApproximation(provenWeight) + if err != nil { + s.log.Warnf("nextStateProofVerifier: LnIntApproximation(%d): %v", provenWeight, err) + return res + } + + return &StateProofVerificationContext{ + LastRound: nextLastRound, + LnProvenWeight: lnProvenWt, + VotersCommitment: vctx.VotersCommitment, + Proto: proto, + } +} + +// SetRenaissance sets the "renaissance" parameters for validating state proofs. +func (s *Service) SetRenaissance(r StateProofVerificationContext) { + s.renaissance = &r +} + +// SetRenaissanceFromConfig sets the "renaissance" parameters for validating state +// proofs based on the settings in the specified cfg. +func (s *Service) SetRenaissanceFromConfig(cfg config.Local) { + if cfg.RenaissanceCatchupRound == 0 { + return + } + + votersCommitment, err := base64.StdEncoding.DecodeString(cfg.RenaissanceCatchupVotersCommitment) + if err != nil { + s.log.Warnf("SetRenaissanceFromConfig: cannot decode voters commitment: %v", err) + return + } + + vc := StateProofVerificationContext{ + LastRound: basics.Round(cfg.RenaissanceCatchupRound), + LnProvenWeight: cfg.RenaissanceCatchupLnProvenWeight, + VotersCommitment: votersCommitment, + Proto: protocol.ConsensusVersion(cfg.RenaissanceCatchupProto), + } + + interval := basics.Round(config.Consensus[vc.Proto].StateProofInterval) + if interval == 0 { + s.log.Warnf("SetRenaissanceFromConfig: state proofs not enabled in specified proto %s", vc.Proto) + return + } + + if (vc.LastRound % interval) != 0 { + s.log.Warnf("SetRenaissanceFromConfig: round %d not multiple of state proof interval %d", vc.LastRound, interval) + return + } + + s.SetRenaissance(vc) +} + +func (s *Service) stateProofWaitEnable() { + s.stateproofmu.Lock() + defer s.stateproofmu.Unlock() + + s.stateproofwait = make(map[basics.Round]chan struct{}) +} + +func (s *Service) stateProofWaitDisable() { + s.stateproofmu.Lock() + defer s.stateproofmu.Unlock() + + for _, ch := range s.stateproofwait { + close(ch) + } + s.stateproofwait = nil +} + +func (s *Service) stateProofWait(r basics.Round) chan struct{} { + s.stateproofmu.Lock() + defer s.stateproofmu.Unlock() + + if r <= s.stateproofmax { + ch := make(chan struct{}) + close(ch) + return ch + } + + if s.stateproofwait == nil { + ch := make(chan struct{}) + close(ch) + return ch + } + + ch, ok := s.stateproofwait[r] + if !ok { + ch = make(chan struct{}) + s.stateproofwait[r] = ch + } + + return ch +} + +func (s *Service) getStateProof(r basics.Round) *stateProofInfo { + s.stateproofmu.Lock() + defer s.stateproofmu.Unlock() + + interval := config.Consensus[s.stateproofproto].StateProofInterval + if interval == 0 { + return nil + } + + proofrnd := r.RoundUpToMultipleOf(basics.Round(interval)) + proofInfo, ok := s.stateproofs[proofrnd] + if !ok { + return nil + } + + return &proofInfo +} + +func (s *Service) startStateProofFetcher(ctx context.Context) { + s.stateProofWaitEnable() + s.workers.Add(1) + go s.stateProofFetcher(ctx) +} + +// stateProofFetcher repeatedly tries to fetch the next verifiable state proof, +// until cancelled or no more state proofs can be fetched. +// +// The caller must s.workers.Add(1) and s.stateProofWaitEnable() before spawning +// stateProofFetcher. +func (s *Service) stateProofFetcher(ctx context.Context) { + defer s.workers.Done() + defer s.stateProofWaitDisable() + + latest := s.ledger.LastRound() + s.cleanupStateProofs(latest) + + peerSelector := createPeerSelector(s.net, s.cfg, true) + retry := 0 + + for { + vc := s.nextStateProofVerifier() + if vc == nil { + s.log.Debugf("catchup.stateProofFetcher: no verifier available") + return + } + + if retry >= catchupRetryLimit { + s.log.Debugf("catchup.stateProofFetcher: cannot fetch %d, giving up", vc.LastRound) + return + } + retry++ + + select { + case <-ctx.Done(): + s.log.Debugf("catchup.stateProofFetcher: aborted") + return + default: + } + + psp, err := peerSelector.getNextPeer() + if err != nil { + s.log.Warnf("catchup.stateProofFetcher: unable to getNextPeer: %v", err) + return + } + + fetcher := makeUniversalBlockFetcher(s.log, s.net, s.cfg) + proofs, _, err := fetcher.fetchStateProof(ctx, protocol.StateProofBasic, vc.LastRound, psp.Peer) + if err != nil { + s.log.Warnf("catchup.fetchStateProof(%d): attempt %d: %v", vc.LastRound, retry, err) + peerSelector.rankPeer(psp, peerRankDownloadFailed) + continue + } + + if len(proofs.Proofs) == 0 { + s.log.Warnf("catchup.fetchStateProof(%d): attempt %d: no proofs returned", vc.LastRound, retry) + peerSelector.rankPeer(psp, peerRankDownloadFailed) + continue + } + + for idx, pf := range proofs.Proofs { + if idx > 0 { + // This is an extra state proof returned optimistically by the server. + // We need to get the corresponding verification context. + vc = s.nextStateProofVerifier() + if vc == nil { + break + } + } + + verifier := stateproof.MkVerifierWithLnProvenWeight(vc.VotersCommitment, vc.LnProvenWeight, config.Consensus[vc.Proto].StateProofStrengthTarget) + err = verifier.Verify(uint64(vc.LastRound), pf.Message.Hash(), &pf.StateProof) + if err != nil { + s.log.Warnf("catchup.stateProofFetcher: cannot verify round %d: %v", vc.LastRound, err) + peerSelector.rankPeer(psp, peerRankInvalidDownload) + break + } + + s.log.Debugf("catchup.stateProofFetcher: validated proof for %d", vc.LastRound) + s.addStateProof(pf.Message, vc.Proto) + retry = 0 + } + } +} + +func verifyBlockStateProof(r basics.Round, spmsg *stateproofmsg.Message, block *bookkeeping.Block, proofData []byte) error { + l := block.ToLightBlockHeader() + + if !config.Consensus[block.CurrentProtocol].StateProofBlockHashInLightHeader { + return fmt.Errorf("block %d protocol %s does not authenticate block in light block header", r, block.CurrentProtocol) + } + + proof, err := merklearray.ProofDataToSingleLeafProof(crypto.Sha256.String(), proofData) + if err != nil { + return err + } + + elems := make(map[uint64]crypto.Hashable) + elems[uint64(r-spmsg.FirstAttestedRound)] = &l + + return merklearray.VerifyVectorCommitment(spmsg.BlockHeadersCommitment, elems, proof.ToProof()) +} diff --git a/catchup/stateproof_test.go b/catchup/stateproof_test.go new file mode 100644 index 0000000000..22bc7dff4d --- /dev/null +++ b/catchup/stateproof_test.go @@ -0,0 +1,173 @@ +// Copyright (C) 2019-2023 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see . + +package catchup + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/algorand/go-algorand/config" + "github.com/algorand/go-algorand/crypto/stateproof" + "github.com/algorand/go-algorand/data/basics" + "github.com/algorand/go-algorand/data/bookkeeping" + "github.com/algorand/go-algorand/logging" + "github.com/algorand/go-algorand/protocol" + "github.com/algorand/go-algorand/rpcs" + "github.com/algorand/go-algorand/test/partitiontest" +) + +func TestServiceStateProofFetcherRenaissance(t *testing.T) { + partitiontest.PartitionTest(t) + + // Make Ledgers + remote, _, blk, spdata, err := buildTestLedger(t, bookkeeping.Block{}) + if err != nil { + t.Fatal(err) + return + } + addBlocks(t, remote, blk, spdata, 1000) + + local := new(mockedLedger) + local.blocks = append(local.blocks, bookkeeping.Block{}) + + // Create a network and block service + blockServiceConfig := config.GetDefaultLocal() + net := &httpTestPeerSource{} + ls := rpcs.MakeBlockService(logging.Base(), blockServiceConfig, remote, net, "test genesisID") + + nodeA := basicRPCNode{} + ls.RegisterHandlers(&nodeA) + nodeA.start() + defer nodeA.stop() + rootURL := nodeA.rootURL() + net.addPeer(rootURL) + + // Make Service + syncer := MakeService(logging.Base(), defaultConfig, net, local, &mockedAuthenticator{errorRound: -1}, nil, nil, nil) + syncer.testStart() + + provenWeight, overflowed := basics.Muldiv(spdata.TotalWeight.ToUint64(), uint64(spdata.Params.StateProofWeightThreshold), 1<<32) + require.False(t, overflowed) + + lnProvenWt, err := stateproof.LnIntApproximation(provenWeight) + require.NoError(t, err) + + syncer.SetRenaissance(StateProofVerificationContext{ + LastRound: 256, + LnProvenWeight: lnProvenWt, + VotersCommitment: spdata.Tree.Root(), + Proto: blk.CurrentProtocol, + }) + + ctx := context.Background() + syncer.startStateProofFetcher(ctx) + + ch := syncer.stateProofWait(500) + <-ch + + msg := syncer.getStateProof(500) + require.NotNil(t, msg) + + ch = syncer.stateProofWait(5000) + <-ch + + msg = syncer.getStateProof(5000) + require.Nil(t, msg) +} + +func TestServiceStateProofSync(t *testing.T) { + partitiontest.PartitionTest(t) + + // Make Ledgers + var blk0 bookkeeping.Block + blk0.CurrentProtocol = protocol.ConsensusFuture + remote, _, blk, spdata, err := buildTestLedger(t, blk0) + if err != nil { + t.Fatal(err) + return + } + addBlocks(t, remote, blk, spdata, 1000) + + local := new(mockedLedger) + local.blocks = append(local.blocks, bookkeeping.Block{}) + + // Create a network and block service + blockServiceConfig := config.GetDefaultLocal() + net := &httpTestPeerSource{} + ls := rpcs.MakeBlockService(logging.Base(), blockServiceConfig, remote, net, "test genesisID") + + nodeA := basicRPCNode{} + ls.RegisterHandlers(&nodeA) + nodeA.start() + defer nodeA.stop() + rootURL := nodeA.rootURL() + net.addPeer(rootURL) + + // Make Service + syncer := MakeService(logging.Base(), defaultConfig, net, local, &mockedAuthenticator{errorRound: -1}, nil, nil, nil) + syncer.testStart() + + provenWeight, overflowed := basics.Muldiv(spdata.TotalWeight.ToUint64(), uint64(spdata.Params.StateProofWeightThreshold), 1<<32) + require.False(t, overflowed) + + lnProvenWt, err := stateproof.LnIntApproximation(provenWeight) + require.NoError(t, err) + + syncer.SetRenaissance(StateProofVerificationContext{ + LastRound: 256, + LnProvenWeight: lnProvenWt, + VotersCommitment: spdata.Tree.Root(), + Proto: blk.CurrentProtocol, + }) + + syncer.sync() + + rr, lr := remote.LastRound(), local.LastRound() + require.Equal(t, rr, lr) + + // Block 500 should have been fetched using state proofs, which means + // we should have no cert in the local ledger. + _, cert, err := local.BlockCert(500) + require.NoError(t, err) + require.True(t, cert.MsgIsZero()) + + // Block 900 should have been fetched using certs, which means + // we should have a valid cert for it in the ledger. + _, cert, err = local.BlockCert(900) + require.NoError(t, err) + require.Equal(t, cert.Round, basics.Round(900)) + + // Now try to sync again, which should flush all of the state proofs already + // covered by the local ledger (which is ahead of the state proofs now). + syncer.sync() + + // Now extend the remote ledger, and make sure that the local ledger can sync + // using state proofs starting from the most recent ledger-generated state + // proof verification context. + addBlocks(t, remote, spdata.TemplateBlock, spdata, 1000) + syncer.sync() + + _, cert, err = local.BlockCert(1500) + require.NoError(t, err) + require.True(t, cert.MsgIsZero()) + + _, cert, err = local.BlockCert(1900) + require.NoError(t, err) + require.Equal(t, cert.Round, basics.Round(1900)) +} diff --git a/catchup/universalFetcher.go b/catchup/universalFetcher.go index c8dd8b9f9f..f157728ecd 100644 --- a/catchup/universalFetcher.go +++ b/catchup/universalFetcher.go @@ -52,8 +52,8 @@ func makeUniversalBlockFetcher(log logging.Logger, net network.GossipNode, confi } // fetchBlock returns a block from the peer. The peer can be either an http or ws peer. -func (uf *universalBlockFetcher) fetchBlock(ctx context.Context, round basics.Round, peer network.Peer) (blk *bookkeeping.Block, - cert *agreement.Certificate, downloadDuration time.Duration, err error) { +func (uf *universalBlockFetcher) fetchBlock(ctx context.Context, round basics.Round, peer network.Peer, proofOK bool) (blk *bookkeeping.Block, + cert *agreement.Certificate, proof []byte, downloadDuration time.Duration, err error) { var fetchedBuf []byte var address string @@ -65,7 +65,7 @@ func (uf *universalBlockFetcher) fetchBlock(ctx context.Context, round basics.Ro } fetchedBuf, err = fetcherClient.getBlockBytes(ctx, round) if err != nil { - return nil, nil, time.Duration(0), err + return nil, nil, nil, time.Duration(0), err } address = fetcherClient.address() } else if httpPeer, validHTTPPeer := peer.(network.HTTPPeer); validHTTPPeer { @@ -76,24 +76,24 @@ func (uf *universalBlockFetcher) fetchBlock(ctx context.Context, round basics.Ro client: httpPeer.GetHTTPClient(), log: uf.log, config: &uf.config} - fetchedBuf, err = fetcherClient.getBlockBytes(ctx, round) + fetchedBuf, err = fetcherClient.getBlockBytes(ctx, round, proofOK) if err != nil { - return nil, nil, time.Duration(0), err + return nil, nil, nil, time.Duration(0), err } address = fetcherClient.address() } else { - return nil, nil, time.Duration(0), fmt.Errorf("fetchBlock: UniversalFetcher only supports HTTPPeer and UnicastPeer") + return nil, nil, nil, time.Duration(0), fmt.Errorf("fetchBlock: UniversalFetcher only supports HTTPPeer and UnicastPeer") } - downloadDuration = time.Now().Sub(blockDownloadStartTime) - block, cert, err := processBlockBytes(fetchedBuf, round, address) + downloadDuration = time.Since(blockDownloadStartTime) + block, cert, proof, err := processBlockBytes(fetchedBuf, round, address) if err != nil { - return nil, nil, time.Duration(0), err + return nil, nil, nil, time.Duration(0), err } uf.log.Debugf("fetchBlock: downloaded block %d in %d from %s", uint64(round), downloadDuration, address) - return block, cert, downloadDuration, err + return block, cert, proof, downloadDuration, err } -func processBlockBytes(fetchedBuf []byte, r basics.Round, peerAddr string) (blk *bookkeeping.Block, cert *agreement.Certificate, err error) { +func processBlockBytes(fetchedBuf []byte, r basics.Round, peerAddr string) (blk *bookkeeping.Block, cert *agreement.Certificate, proof []byte, err error) { var decodedEntry rpcs.EncodedBlockCert err = protocol.Decode(fetchedBuf, &decodedEntry) if err != nil { @@ -106,11 +106,56 @@ func processBlockBytes(fetchedBuf []byte, r basics.Round, peerAddr string) (blk return } - if decodedEntry.Certificate.Round != r { + if (decodedEntry.LightBlockHeaderProof == nil || decodedEntry.Certificate.Round != 0) && decodedEntry.Certificate.Round != r { err = makeErrWrongCertFromPeer(r, decodedEntry.Certificate.Round, peerAddr) return } - return &decodedEntry.Block, &decodedEntry.Certificate, nil + return &decodedEntry.Block, &decodedEntry.Certificate, decodedEntry.LightBlockHeaderProof, nil +} + +// fetchStateProof retrieves a state proof (and the corresponding message +// attested to by the state proof) for round. +// +// proofType specifies the expected state proof type. +func (uf *universalBlockFetcher) fetchStateProof(ctx context.Context, proofType protocol.StateProofType, round basics.Round, peer network.Peer) (proofs rpcs.StateProofResponse, downloadDuration time.Duration, err error) { + var fetchedBuf []byte + var address string + downloadStartTime := time.Now() + if httpPeer, validHTTPPeer := peer.(network.HTTPPeer); validHTTPPeer { + fetcherClient := &HTTPFetcher{ + peer: httpPeer, + rootURL: httpPeer.GetAddress(), + net: uf.net, + client: httpPeer.GetHTTPClient(), + log: uf.log, + config: &uf.config, + } + fetchedBuf, err = fetcherClient.getStateProofBytes(ctx, proofType, round) + if err != nil { + return proofs, 0, err + } + address = fetcherClient.address() + } else { + return proofs, 0, fmt.Errorf("fetchStateProof: UniversalFetcher only supports HTTPPeer") + } + downloadDuration = time.Since(downloadStartTime) + proofs, err = processStateProofBytes(fetchedBuf, round, address) + if err != nil { + return proofs, 0, err + } + uf.log.Debugf("fetchStateProof: downloaded proof for %d in %d from %s", uint64(round), downloadDuration, address) + return proofs, downloadDuration, err +} + +func processStateProofBytes(fetchedBuf []byte, r basics.Round, peerAddr string) (proofs rpcs.StateProofResponse, err error) { + var decodedEntry rpcs.StateProofResponse + err = protocol.Decode(fetchedBuf, &decodedEntry) + if err != nil { + err = fmt.Errorf("Cannot decode state proof for %d from %s: %v", r, peerAddr, err) + return + } + + return decodedEntry, nil } // a stub fetcherClient to satisfy the NetworkFetcher interface @@ -209,18 +254,10 @@ type HTTPFetcher struct { config *config.Local } -// getBlockBytes gets a block. -// Core piece of FetcherClient interface -func (hf *HTTPFetcher) getBlockBytes(ctx context.Context, r basics.Round) (data []byte, err error) { - parsedURL, err := network.ParseHostOrURL(hf.rootURL) - if err != nil { - return nil, err - } - - parsedURL.Path = rpcs.FormatBlockQuery(uint64(r), parsedURL.Path, hf.net) - blockURL := parsedURL.String() - hf.log.Debugf("block GET %#v peer %#v %T", blockURL, hf.peer, hf.peer) - request, err := http.NewRequest("GET", blockURL, nil) +// getBytes requests a particular URL, expecting specific content types +func (hf *HTTPFetcher) getBytes(ctx context.Context, url string, expectedContentTypes map[string]struct{}) (data []byte, err error) { + hf.log.Debugf("block GET %#v peer %#v %T", url, hf.peer, hf.peer) + request, err := http.NewRequest("GET", url, nil) if err != nil { return nil, err } @@ -230,7 +267,7 @@ func (hf *HTTPFetcher) getBlockBytes(ctx context.Context, r basics.Round) (data network.SetUserAgentHeader(request.Header) response, err := hf.client.Do(request) if err != nil { - hf.log.Debugf("GET %#v : %s", blockURL, err) + hf.log.Debugf("GET %#v : %s", url, err) return nil, err } @@ -242,11 +279,11 @@ func (hf *HTTPFetcher) getBlockBytes(ctx context.Context, r basics.Round) (data return nil, errNoBlockForRound default: bodyBytes, err := rpcs.ResponseBytes(response, hf.log, fetcherMaxBlockBytes) - hf.log.Warnf("HTTPFetcher.getBlockBytes: response status code %d from '%s'. Response body '%s' ", response.StatusCode, blockURL, string(bodyBytes)) + hf.log.Warnf("HTTPFetcher.getBytes: response status code %d from '%s'. Response body '%s' ", response.StatusCode, url, string(bodyBytes)) if err == nil { - err = makeErrHTTPResponse(response.StatusCode, blockURL, fmt.Sprintf("Response body '%s'", string(bodyBytes))) + err = makeErrHTTPResponse(response.StatusCode, url, fmt.Sprintf("Response body '%s'", string(bodyBytes))) } else { - err = makeErrHTTPResponse(response.StatusCode, blockURL, err.Error()) + err = makeErrHTTPResponse(response.StatusCode, url, err.Error()) } return nil, err } @@ -261,11 +298,9 @@ func (hf *HTTPFetcher) getBlockBytes(ctx context.Context, r basics.Round) (data return nil, err } - // TODO: Temporarily allow old and new content types so we have time for lazy upgrades - // Remove this 'old' string after next release. - const blockResponseContentTypeOld = "application/algorand-block-v1" - if contentTypes[0] != rpcs.BlockResponseContentType && contentTypes[0] != blockResponseContentTypeOld { - hf.log.Warnf("http block fetcher response has an invalid content type : %s", contentTypes[0]) + _, contentTypeOK := expectedContentTypes[contentTypes[0]] + if !contentTypeOK { + hf.log.Warnf("http fetcher response has an invalid content type : %s", contentTypes[0]) response.Body.Close() return nil, errHTTPResponseContentType{contentTypeCount: 1, contentType: contentTypes[0]} } @@ -273,6 +308,48 @@ func (hf *HTTPFetcher) getBlockBytes(ctx context.Context, r basics.Round) (data return rpcs.ResponseBytes(response, hf.log, fetcherMaxBlockBytes) } +// getBlockBytes gets a block. +// Core piece of FetcherClient interface +func (hf *HTTPFetcher) getBlockBytes(ctx context.Context, r basics.Round, proofOK bool) (data []byte, err error) { + parsedURL, err := network.ParseHostOrURL(hf.rootURL) + if err != nil { + return nil, err + } + + parsedURL.Path = rpcs.FormatBlockQuery(uint64(r), parsedURL.Path, hf.net) + if proofOK { + parsedURL.RawQuery = "stateproof=0" + } + blockURL := parsedURL.String() + + // TODO: Temporarily allow old and new content types so we have time for lazy upgrades + // Remove this 'old' string after next release. + const blockResponseContentTypeOld = "application/algorand-block-v1" + expectedContentTypes := map[string]struct{}{ + rpcs.BlockResponseContentType: {}, + blockResponseContentTypeOld: {}, + } + + return hf.getBytes(ctx, blockURL, expectedContentTypes) +} + +// getStateProofBytes gets a state proof. +func (hf *HTTPFetcher) getStateProofBytes(ctx context.Context, proofType protocol.StateProofType, r basics.Round) (data []byte, err error) { + parsedURL, err := network.ParseHostOrURL(hf.rootURL) + if err != nil { + return nil, err + } + + parsedURL.Path = rpcs.FormatStateProofQuery(uint64(r), proofType, parsedURL.Path, hf.net) + proofURL := parsedURL.String() + + expectedContentTypes := map[string]struct{}{ + rpcs.StateProofResponseContentType: {}, + } + + return hf.getBytes(ctx, proofURL, expectedContentTypes) +} + // Address is part of FetcherClient interface. // Returns the root URL of the connected peer. func (hf *HTTPFetcher) address() string { diff --git a/catchup/universalFetcher_test.go b/catchup/universalFetcher_test.go index 4414bb9c11..34a6eef81f 100644 --- a/catchup/universalFetcher_test.go +++ b/catchup/universalFetcher_test.go @@ -44,7 +44,7 @@ func TestUGetBlockWs(t *testing.T) { cfg := config.GetDefaultLocal() - ledger, next, b, err := buildTestLedger(t, bookkeeping.Block{}) + ledger, next, b, _, err := buildTestLedger(t, bookkeeping.Block{}) if err != nil { t.Fatal(err) return @@ -65,13 +65,13 @@ func TestUGetBlockWs(t *testing.T) { var cert *agreement.Certificate var duration time.Duration - block, cert, _, err = fetcher.fetchBlock(context.Background(), next, up) + block, cert, _, _, err = fetcher.fetchBlock(context.Background(), next, up, false) require.NoError(t, err) require.Equal(t, &b, block) require.GreaterOrEqual(t, int64(duration), int64(0)) - block, cert, duration, err = fetcher.fetchBlock(context.Background(), next+1, up) + block, cert, _, duration, err = fetcher.fetchBlock(context.Background(), next+1, up, false) require.Error(t, err) require.Contains(t, err.Error(), "requested block is not available") @@ -86,7 +86,7 @@ func TestUGetBlockHTTP(t *testing.T) { cfg := config.GetDefaultLocal() - ledger, next, b, err := buildTestLedger(t, bookkeeping.Block{}) + ledger, next, b, _, err := buildTestLedger(t, bookkeeping.Block{}) if err != nil { t.Fatal(err) return @@ -111,13 +111,13 @@ func TestUGetBlockHTTP(t *testing.T) { var block *bookkeeping.Block var cert *agreement.Certificate var duration time.Duration - block, cert, duration, err = fetcher.fetchBlock(context.Background(), next, net.GetPeers()[0]) + block, cert, _, duration, err = fetcher.fetchBlock(context.Background(), next, net.GetPeers()[0], false) require.NoError(t, err) require.Equal(t, &b, block) require.GreaterOrEqual(t, int64(duration), int64(0)) - block, cert, duration, err = fetcher.fetchBlock(context.Background(), next+1, net.GetPeers()[0]) + block, cert, _, duration, err = fetcher.fetchBlock(context.Background(), next+1, net.GetPeers()[0], false) require.Error(t, errNoBlockForRound, err) require.Contains(t, err.Error(), "No block available for given round") @@ -132,7 +132,7 @@ func TestUGetBlockUnsupported(t *testing.T) { fetcher := universalBlockFetcher{} peer := "" - block, cert, duration, err := fetcher.fetchBlock(context.Background(), 1, peer) + block, cert, _, duration, err := fetcher.fetchBlock(context.Background(), 1, peer, false) require.Error(t, err) require.Contains(t, err.Error(), "fetchBlock: UniversalFetcher only supports HTTPPeer and UnicastPeer") require.Nil(t, block) @@ -156,18 +156,18 @@ func TestProcessBlockBytesErrors(t *testing.T) { }) // Check for cert error - _, _, err := processBlockBytes(bc, 22, "test") + _, _, _, err := processBlockBytes(bc, 22, "test") var wcfpe errWrongCertFromPeer require.True(t, errors.As(err, &wcfpe)) // Check for round error - _, _, err = processBlockBytes(bc, 20, "test") + _, _, _, err = processBlockBytes(bc, 20, "test") var wbfpe errWrongBlockFromPeer require.True(t, errors.As(err, &wbfpe)) // Check for undecodable bc[11] = 0 - _, _, err = processBlockBytes(bc, 22, "test") + _, _, _, err = processBlockBytes(bc, 22, "test") var cdbe errCannotDecodeBlock require.True(t, errors.As(err, &cdbe)) } @@ -178,7 +178,7 @@ func TestRequestBlockBytesErrors(t *testing.T) { cfg := config.GetDefaultLocal() - ledger, next, _, err := buildTestLedger(t, bookkeeping.Block{}) + ledger, next, _, _, err := buildTestLedger(t, bookkeeping.Block{}) if err != nil { t.Fatal(err) return @@ -199,7 +199,7 @@ func TestRequestBlockBytesErrors(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) cancel() - _, _, _, err = fetcher.fetchBlock(ctx, next, up) + _, _, _, _, err = fetcher.fetchBlock(ctx, next, up, false) var wrfe errWsFetcherRequestFailed require.True(t, errors.As(err, &wrfe), "unexpected err: %w", wrfe) require.Equal(t, "context canceled", err.(errWsFetcherRequestFailed).cause) @@ -209,14 +209,14 @@ func TestRequestBlockBytesErrors(t *testing.T) { responseOverride := network.Response{Topics: network.Topics{network.MakeTopic(rpcs.BlockDataKey, make([]byte, 0))}} up = makeTestUnicastPeerWithResponseOverride(net, t, &responseOverride) - _, _, _, err = fetcher.fetchBlock(ctx, next, up) + _, _, _, _, err = fetcher.fetchBlock(ctx, next, up, false) require.True(t, errors.As(err, &wrfe)) require.Equal(t, "Cert data not found", err.(errWsFetcherRequestFailed).cause) responseOverride = network.Response{Topics: network.Topics{network.MakeTopic(rpcs.CertDataKey, make([]byte, 0))}} up = makeTestUnicastPeerWithResponseOverride(net, t, &responseOverride) - _, _, _, err = fetcher.fetchBlock(ctx, next, up) + _, _, _, _, err = fetcher.fetchBlock(ctx, next, up, false) require.True(t, errors.As(err, &wrfe)) require.Equal(t, "Block data not found", err.(errWsFetcherRequestFailed).cause) } @@ -259,26 +259,26 @@ func TestGetBlockBytesHTTPErrors(t *testing.T) { fetcher := makeUniversalBlockFetcher(logging.TestingLog(t), net, cfg) ls.status = http.StatusBadRequest - _, _, _, err := fetcher.fetchBlock(context.Background(), 1, net.GetPeers()[0]) + _, _, _, _, err := fetcher.fetchBlock(context.Background(), 1, net.GetPeers()[0], false) var hre errHTTPResponse require.True(t, errors.As(err, &hre)) require.Equal(t, "Response body '\x00'", err.(errHTTPResponse).cause) ls.exceedLimit = true - _, _, _, err = fetcher.fetchBlock(context.Background(), 1, net.GetPeers()[0]) + _, _, _, _, err = fetcher.fetchBlock(context.Background(), 1, net.GetPeers()[0], false) require.True(t, errors.As(err, &hre)) require.Equal(t, "read limit exceeded", err.(errHTTPResponse).cause) ls.status = http.StatusOK ls.content = append(ls.content, "undefined") - _, _, _, err = fetcher.fetchBlock(context.Background(), 1, net.GetPeers()[0]) + _, _, _, _, err = fetcher.fetchBlock(context.Background(), 1, net.GetPeers()[0], false) var cte errHTTPResponseContentType require.True(t, errors.As(err, &cte)) require.Equal(t, "undefined", err.(errHTTPResponseContentType).contentType) ls.status = http.StatusOK ls.content = append(ls.content, "undefined2") - _, _, _, err = fetcher.fetchBlock(context.Background(), 1, net.GetPeers()[0]) + _, _, _, _, err = fetcher.fetchBlock(context.Background(), 1, net.GetPeers()[0], false) require.True(t, errors.As(err, &cte)) require.Equal(t, 2, err.(errHTTPResponseContentType).contentTypeCount) } diff --git a/components/mocks/mockNetwork.go b/components/mocks/mockNetwork.go index 8c8eb113f1..5d47e6726f 100644 --- a/components/mocks/mockNetwork.go +++ b/components/mocks/mockNetwork.go @@ -98,6 +98,10 @@ func (network *MockNetwork) ClearHandlers() { func (network *MockNetwork) RegisterHTTPHandler(path string, handler http.Handler) { } +// RegisterHTTPHandlerFunc - empty implementation +func (network *MockNetwork) RegisterHTTPHandlerFunc(path string, handler func(response http.ResponseWriter, request *http.Request)) { +} + // OnNetworkAdvance - empty implementation func (network *MockNetwork) OnNetworkAdvance() {} diff --git a/config/config.go b/config/config.go index fc2dd30050..6ee70213b2 100644 --- a/config/config.go +++ b/config/config.go @@ -72,6 +72,11 @@ const StateProofFileName = "stateproof.sqlite" // It is used for tracking participation key metadata. const ParticipationRegistryFilename = "partregistry.sqlite" +// StateProofCatchupFilename is the name of the database file that is used +// during catchup to temporarily store validated state proofs for blocks +// ahead of the ledger. +const StateProofCatchupFilename = "stateproofcatchup.sqlite" + // ConfigurableConsensusProtocolsFilename defines a set of consensus protocols that // are to be loaded from the data directory ( if present ), to override the // built-in supported consensus protocols. diff --git a/config/config_test.go b/config/config_test.go index c88b53834e..ac1041947e 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -687,6 +687,7 @@ func TestEnsureAndResolveGenesisDirs(t *testing.T) { cfg.BlockDBDir = filepath.Join(testDirectory, "/BAD/BAD/../../custom_block") cfg.CrashDBDir = filepath.Join(testDirectory, "custom_crash") cfg.StateproofDir = filepath.Join(testDirectory, "/RELATIVEPATHS/../RELATIVE/../custom_stateproof") + cfg.StateproofCatchupDir = filepath.Join(testDirectory, "/RELATIVEPATHS/../RELATIVE/../custom_stateproof_catchup") cfg.CatchpointDir = filepath.Join(testDirectory, "custom_catchpoint") paths, err := cfg.EnsureAndResolveGenesisDirs(testDirectory, "myGenesisID") @@ -701,6 +702,8 @@ func TestEnsureAndResolveGenesisDirs(t *testing.T) { require.DirExists(t, paths.CrashGenesisDir) require.Equal(t, testDirectory+"/custom_stateproof/myGenesisID", paths.StateproofGenesisDir) require.DirExists(t, paths.StateproofGenesisDir) + require.Equal(t, testDirectory+"/custom_stateproof_catchup/myGenesisID", paths.StateproofCatchupGenesisDir) + require.DirExists(t, paths.StateproofCatchupGenesisDir) require.Equal(t, testDirectory+"/custom_catchpoint/myGenesisID", paths.CatchpointGenesisDir) require.DirExists(t, paths.CatchpointGenesisDir) } @@ -722,6 +725,8 @@ func TestEnsureAndResolveGenesisDirs_hierarchy(t *testing.T) { require.DirExists(t, paths.CrashGenesisDir) require.Equal(t, testDirectory+"/myGenesisID", paths.StateproofGenesisDir) require.DirExists(t, paths.StateproofGenesisDir) + require.Equal(t, testDirectory+"/myGenesisID", paths.StateproofCatchupGenesisDir) + require.DirExists(t, paths.StateproofCatchupGenesisDir) require.Equal(t, testDirectory+"/myGenesisID", paths.CatchpointGenesisDir) require.DirExists(t, paths.CatchpointGenesisDir) @@ -742,6 +747,8 @@ func TestEnsureAndResolveGenesisDirs_hierarchy(t *testing.T) { require.DirExists(t, paths.CrashGenesisDir) require.Equal(t, cold+"/myGenesisID", paths.StateproofGenesisDir) require.DirExists(t, paths.StateproofGenesisDir) + require.Equal(t, cold+"/myGenesisID", paths.StateproofCatchupGenesisDir) + require.DirExists(t, paths.StateproofCatchupGenesisDir) require.Equal(t, cold+"/myGenesisID", paths.CatchpointGenesisDir) require.DirExists(t, paths.CatchpointGenesisDir) } @@ -758,6 +765,7 @@ func TestEnsureAndResolveGenesisDirsError(t *testing.T) { cfg.BlockDBDir = filepath.Join(testDirectory, "/BAD/BAD/../../custom_block") cfg.CrashDBDir = filepath.Join(testDirectory, "custom_crash") cfg.StateproofDir = filepath.Join(testDirectory, "/RELATIVEPATHS/../RELATIVE/../custom_stateproof") + cfg.StateproofCatchupDir = filepath.Join(testDirectory, "/RELATIVEPATHS/../RELATIVE/../custom_stateproof_catchup") cfg.CatchpointDir = filepath.Join(testDirectory, "custom_catchpoint") // first try an error with an empty root dir diff --git a/config/localTemplate.go b/config/localTemplate.go index 4202c2648a..9d037e82b4 100644 --- a/config/localTemplate.go +++ b/config/localTemplate.go @@ -42,7 +42,7 @@ type Local struct { // Version tracks the current version of the defaults so we can migrate old -> new // This is specifically important whenever we decide to change the default value // for an existing parameter. This field tag must be updated any time we add a new version. - Version uint32 `version[0]:"0" version[1]:"1" version[2]:"2" version[3]:"3" version[4]:"4" version[5]:"5" version[6]:"6" version[7]:"7" version[8]:"8" version[9]:"9" version[10]:"10" version[11]:"11" version[12]:"12" version[13]:"13" version[14]:"14" version[15]:"15" version[16]:"16" version[17]:"17" version[18]:"18" version[19]:"19" version[20]:"20" version[21]:"21" version[22]:"22" version[23]:"23" version[24]:"24" version[25]:"25" version[26]:"26" version[27]:"27" version[28]:"28" version[29]:"29" version[30]:"30" version[31]:"31"` + Version uint32 `version[0]:"0" version[1]:"1" version[2]:"2" version[3]:"3" version[4]:"4" version[5]:"5" version[6]:"6" version[7]:"7" version[8]:"8" version[9]:"9" version[10]:"10" version[11]:"11" version[12]:"12" version[13]:"13" version[14]:"14" version[15]:"15" version[16]:"16" version[17]:"17" version[18]:"18" version[19]:"19" version[20]:"20" version[21]:"21" version[22]:"22" version[23]:"23" version[24]:"24" version[25]:"25" version[26]:"26" version[27]:"27" version[28]:"28" version[29]:"29" version[30]:"30" version[31]:"31" version[32]:"32"` // environmental (may be overridden) // When enabled, stores blocks indefinitely, otherwise, only the most recent blocks @@ -111,6 +111,10 @@ type Local struct { // For isolation, the node will create a subdirectory in this location, named by the genesis-id of the network. // If not specified, the node will use the ColdDataDir. StateproofDir string `version[31]:""` + // StateproofCatchupDir is an optional directory to store stateproof catchup data. + // For isolation, the node will create a subdirectory in this location, named by the genesis-id of the network. + // If not specified, the node will use the ColdDataDir. + StateproofCatchupDir string `version[31]:""` // CrashDBDir is an optional directory to store the crash database. // For isolation, the node will create a subdirectory in this location, named by the genesis-id of the network. // If not specified, the node will use the ColdDataDir. @@ -581,6 +585,25 @@ type Local struct { // DisableAPIAuth turns off authentication for public (non-admin) API endpoints. DisableAPIAuth bool `version[30]:"false"` + + // RenaissanceCatchup* fields specify how to bootstrap authentication of + // state proofs without validating every block starting from genesis. + // + // RenaissanceCatchupRound is the next expected LastAttestedRound in the + // state proof we expect to verify using these renaissance parameters. + // If 0 (default), renaissance catchup parameters are disabled. + RenaissanceCatchupRound uint64 `version[32]:"0"` + + // RenaissanceCatchupLnProvenWeight is the corresponding field from the + // previous state proof message. + RenaissanceCatchupLnProvenWeight uint64 `version[32]:"0"` + + // RenaissanceCatchupVotersCommitment is the base64-encoded corresponding + // field from the previous state proof message. + RenaissanceCatchupVotersCommitment string `version[32]:""` + + // RenaissanceCatchupProto is the protocol of RenaissanceCatchupRound. + RenaissanceCatchupProto string `version[32]:""` } // DNSBootstrapArray returns an array of one or more DNS Bootstrap identifiers @@ -708,14 +731,15 @@ func ensureAbsGenesisDir(path string, genesisID string) (string, error) { // ResolvedGenesisDirs is a collection of directories including Genesis ID // Subdirectories for execution of a node type ResolvedGenesisDirs struct { - RootGenesisDir string - HotGenesisDir string - ColdGenesisDir string - TrackerGenesisDir string - BlockGenesisDir string - CatchpointGenesisDir string - StateproofGenesisDir string - CrashGenesisDir string + RootGenesisDir string + HotGenesisDir string + ColdGenesisDir string + TrackerGenesisDir string + BlockGenesisDir string + CatchpointGenesisDir string + StateproofGenesisDir string + StateproofCatchupGenesisDir string + CrashGenesisDir string } // String returns the Genesis Directory values as a string @@ -728,6 +752,7 @@ func (rgd ResolvedGenesisDirs) String() string { ret += fmt.Sprintf("BlockGenesisDir: %s\n", rgd.BlockGenesisDir) ret += fmt.Sprintf("CatchpointGenesisDir: %s\n", rgd.CatchpointGenesisDir) ret += fmt.Sprintf("StateproofGenesisDir: %s\n", rgd.StateproofGenesisDir) + ret += fmt.Sprintf("StateproofCatchupGenesisDir: %s\n", rgd.StateproofCatchupGenesisDir) ret += fmt.Sprintf("CrashGenesisDir: %s\n", rgd.CrashGenesisDir) return ret } @@ -823,6 +848,15 @@ func (cfg *Local) EnsureAndResolveGenesisDirs(rootDir, genesisID string) (Resolv } else { resolved.StateproofGenesisDir = resolved.ColdGenesisDir } + // if StateproofCatchupDir is not set, use ColdDataDir + if cfg.StateproofCatchupDir != "" { + resolved.StateproofCatchupGenesisDir, err = ensureAbsGenesisDir(cfg.StateproofCatchupDir, genesisID) + if err != nil { + return ResolvedGenesisDirs{}, err + } + } else { + resolved.StateproofCatchupGenesisDir = resolved.ColdGenesisDir + } // if CrashDBDir is not set, use ColdDataDir if cfg.CrashDBDir != "" { resolved.CrashGenesisDir, err = ensureAbsGenesisDir(cfg.CrashDBDir, genesisID) diff --git a/config/local_defaults.go b/config/local_defaults.go index fd0aa20521..08431f2013 100644 --- a/config/local_defaults.go +++ b/config/local_defaults.go @@ -20,7 +20,7 @@ package config var defaultLocal = Local{ - Version: 31, + Version: 32, AccountUpdatesStatsInterval: 5000000000, AccountsRebuildSynchronousMode: 1, AgreementIncomingBundlesQueueLength: 15, @@ -124,12 +124,17 @@ var defaultLocal = Local{ ProposalAssemblyTime: 500000000, PublicAddress: "", ReconnectTime: 60000000000, + RenaissanceCatchupLnProvenWeight: 0, + RenaissanceCatchupProto: "", + RenaissanceCatchupRound: 0, + RenaissanceCatchupVotersCommitment: "", ReservedFDs: 256, RestConnectionsHardLimit: 2048, RestConnectionsSoftLimit: 1024, RestReadTimeoutSeconds: 15, RestWriteTimeoutSeconds: 120, RunHosted: false, + StateproofCatchupDir: "", StateproofDir: "", StorageEngine: "sqlite", SuggestedFeeBlockHistory: 3, diff --git a/daemon/algod/api/server/v2/handlers.go b/daemon/algod/api/server/v2/handlers.go index 7b190acd52..c330706ba2 100644 --- a/daemon/algod/api/server/v2/handlers.go +++ b/daemon/algod/api/server/v2/handlers.go @@ -213,8 +213,8 @@ func GetStateProofTransactionForRound(ctx context.Context, txnFetcher LedgerForA continue } - if txn.Txn.StateProofTxnFields.Message.FirstAttestedRound <= uint64(round) && - uint64(round) <= txn.Txn.StateProofTxnFields.Message.LastAttestedRound { + if txn.Txn.StateProofTxnFields.Message.FirstAttestedRound <= round && + round <= txn.Txn.StateProofTxnFields.Message.LastAttestedRound { return txn.Txn, nil } } @@ -1685,8 +1685,8 @@ func (v2 *Handlers) GetStateProof(ctx echo.Context, round uint64) error { response.Message.BlockHeadersCommitment = tx.Message.BlockHeadersCommitment response.Message.VotersCommitment = tx.Message.VotersCommitment response.Message.LnProvenWeight = tx.Message.LnProvenWeight - response.Message.FirstAttestedRound = tx.Message.FirstAttestedRound - response.Message.LastAttestedRound = tx.Message.LastAttestedRound + response.Message.FirstAttestedRound = uint64(tx.Message.FirstAttestedRound) + response.Message.LastAttestedRound = uint64(tx.Message.LastAttestedRound) return ctx.JSON(http.StatusOK, response) } @@ -1718,14 +1718,14 @@ func (v2 *Handlers) GetLightBlockHeaderProof(ctx echo.Context, round uint64) err lastAttestedRound := stateProof.Message.LastAttestedRound firstAttestedRound := stateProof.Message.FirstAttestedRound - stateProofInterval := lastAttestedRound - firstAttestedRound + 1 + stateProofInterval := uint64(lastAttestedRound - firstAttestedRound + 1) - lightHeaders, err := stateproof.FetchLightHeaders(ledger, stateProofInterval, basics.Round(lastAttestedRound)) + lightHeaders, err := stateproof.FetchLightHeaders(ledger, stateProofInterval, lastAttestedRound) if err != nil { return notFound(ctx, err, err.Error(), v2.Log) } - blockIndex := round - firstAttestedRound + blockIndex := round - uint64(firstAttestedRound) leafproof, err := stateproof.GenerateProofOfLightBlockHeaders(stateProofInterval, lightHeaders, blockIndex) if err != nil { return internalError(ctx, err, err.Error(), v2.Log) diff --git a/daemon/algod/api/server/v2/test/handlers_test.go b/daemon/algod/api/server/v2/test/handlers_test.go index 48f89538bb..6de13b26cd 100644 --- a/daemon/algod/api/server/v2/test/handlers_test.go +++ b/daemon/algod/api/server/v2/test/handlers_test.go @@ -1872,8 +1872,8 @@ func addStateProof(blk bookkeeping.Block) bookkeeping.Block { StateProofType: 0, Message: stateproofmsg.Message{ BlockHeadersCommitment: []byte{0x0, 0x1, 0x2}, - FirstAttestedRound: stateProofRound + 1, - LastAttestedRound: stateProofRound + stateProofInterval, + FirstAttestedRound: basics.Round(stateProofRound + 1), + LastAttestedRound: basics.Round(stateProofRound + stateProofInterval), }, }, }, diff --git a/data/stateproofmsg/message.go b/data/stateproofmsg/message.go index 5f1c2e3433..4796f0acef 100644 --- a/data/stateproofmsg/message.go +++ b/data/stateproofmsg/message.go @@ -19,6 +19,7 @@ package stateproofmsg import ( "github.com/algorand/go-algorand/crypto" sp "github.com/algorand/go-algorand/crypto/stateproof" + "github.com/algorand/go-algorand/data/basics" "github.com/algorand/go-algorand/protocol" ) @@ -29,11 +30,11 @@ import ( type Message struct { _struct struct{} `codec:",omitempty,omitemptyarray"` // BlockHeadersCommitment contains a commitment on all light block headers within a state proof interval. - BlockHeadersCommitment []byte `codec:"b,allocbound=crypto.Sha256Size"` - VotersCommitment []byte `codec:"v,allocbound=crypto.SumhashDigestSize"` - LnProvenWeight uint64 `codec:"P"` - FirstAttestedRound uint64 `codec:"f"` - LastAttestedRound uint64 `codec:"l"` + BlockHeadersCommitment []byte `codec:"b,allocbound=crypto.Sha256Size"` + VotersCommitment []byte `codec:"v,allocbound=crypto.SumhashDigestSize"` + LnProvenWeight uint64 `codec:"P"` + FirstAttestedRound basics.Round `codec:"f"` + LastAttestedRound basics.Round `codec:"l"` } // ToBeHashed returns the bytes of the message. diff --git a/data/stateproofmsg/msgp_gen.go b/data/stateproofmsg/msgp_gen.go index 02a17491bf..f59f844a32 100644 --- a/data/stateproofmsg/msgp_gen.go +++ b/data/stateproofmsg/msgp_gen.go @@ -6,6 +6,7 @@ import ( "github.com/algorand/msgp/msgp" "github.com/algorand/go-algorand/crypto" + "github.com/algorand/go-algorand/data/basics" ) // The following msgp objects are implemented in this file: @@ -33,11 +34,11 @@ func (z *Message) MarshalMsg(b []byte) (o []byte) { zb0001Len-- zb0001Mask |= 0x4 } - if (*z).FirstAttestedRound == 0 { + if (*z).FirstAttestedRound.MsgIsZero() { zb0001Len-- zb0001Mask |= 0x8 } - if (*z).LastAttestedRound == 0 { + if (*z).LastAttestedRound.MsgIsZero() { zb0001Len-- zb0001Mask |= 0x10 } @@ -61,12 +62,12 @@ func (z *Message) MarshalMsg(b []byte) (o []byte) { if (zb0001Mask & 0x8) == 0 { // if not empty // string "f" o = append(o, 0xa1, 0x66) - o = msgp.AppendUint64(o, (*z).FirstAttestedRound) + o = (*z).FirstAttestedRound.MarshalMsg(o) } if (zb0001Mask & 0x10) == 0 { // if not empty // string "l" o = append(o, 0xa1, 0x6c) - o = msgp.AppendUint64(o, (*z).LastAttestedRound) + o = (*z).LastAttestedRound.MarshalMsg(o) } if (zb0001Mask & 0x20) == 0 { // if not empty // string "v" @@ -141,7 +142,7 @@ func (z *Message) UnmarshalMsg(bts []byte) (o []byte, err error) { } if zb0001 > 0 { zb0001-- - (*z).FirstAttestedRound, bts, err = msgp.ReadUint64Bytes(bts) + bts, err = (*z).FirstAttestedRound.UnmarshalMsg(bts) if err != nil { err = msgp.WrapError(err, "struct-from-array", "FirstAttestedRound") return @@ -149,7 +150,7 @@ func (z *Message) UnmarshalMsg(bts []byte) (o []byte, err error) { } if zb0001 > 0 { zb0001-- - (*z).LastAttestedRound, bts, err = msgp.ReadUint64Bytes(bts) + bts, err = (*z).LastAttestedRound.UnmarshalMsg(bts) if err != nil { err = msgp.WrapError(err, "struct-from-array", "LastAttestedRound") return @@ -217,13 +218,13 @@ func (z *Message) UnmarshalMsg(bts []byte) (o []byte, err error) { return } case "f": - (*z).FirstAttestedRound, bts, err = msgp.ReadUint64Bytes(bts) + bts, err = (*z).FirstAttestedRound.UnmarshalMsg(bts) if err != nil { err = msgp.WrapError(err, "FirstAttestedRound") return } case "l": - (*z).LastAttestedRound, bts, err = msgp.ReadUint64Bytes(bts) + bts, err = (*z).LastAttestedRound.UnmarshalMsg(bts) if err != nil { err = msgp.WrapError(err, "LastAttestedRound") return @@ -248,17 +249,17 @@ func (_ *Message) CanUnmarshalMsg(z interface{}) bool { // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message func (z *Message) Msgsize() (s int) { - s = 1 + 2 + msgp.BytesPrefixSize + len((*z).BlockHeadersCommitment) + 2 + msgp.BytesPrefixSize + len((*z).VotersCommitment) + 2 + msgp.Uint64Size + 2 + msgp.Uint64Size + 2 + msgp.Uint64Size + s = 1 + 2 + msgp.BytesPrefixSize + len((*z).BlockHeadersCommitment) + 2 + msgp.BytesPrefixSize + len((*z).VotersCommitment) + 2 + msgp.Uint64Size + 2 + (*z).FirstAttestedRound.Msgsize() + 2 + (*z).LastAttestedRound.Msgsize() return } // MsgIsZero returns whether this is a zero value func (z *Message) MsgIsZero() bool { - return (len((*z).BlockHeadersCommitment) == 0) && (len((*z).VotersCommitment) == 0) && ((*z).LnProvenWeight == 0) && ((*z).FirstAttestedRound == 0) && ((*z).LastAttestedRound == 0) + return (len((*z).BlockHeadersCommitment) == 0) && (len((*z).VotersCommitment) == 0) && ((*z).LnProvenWeight == 0) && ((*z).FirstAttestedRound.MsgIsZero()) && ((*z).LastAttestedRound.MsgIsZero()) } // MaxSize returns a maximum valid message size for this message type func MessageMaxSize() (s int) { - s = 1 + 2 + msgp.BytesPrefixSize + crypto.Sha256Size + 2 + msgp.BytesPrefixSize + crypto.SumhashDigestSize + 2 + msgp.Uint64Size + 2 + msgp.Uint64Size + 2 + msgp.Uint64Size + s = 1 + 2 + msgp.BytesPrefixSize + crypto.Sha256Size + 2 + msgp.BytesPrefixSize + crypto.SumhashDigestSize + 2 + msgp.Uint64Size + 2 + basics.RoundMaxSize() + 2 + basics.RoundMaxSize() return } diff --git a/installer/config.json.example b/installer/config.json.example index 62cbe6427b..47300a4332 100644 --- a/installer/config.json.example +++ b/installer/config.json.example @@ -1,5 +1,5 @@ { - "Version": 31, + "Version": 32, "AccountUpdatesStatsInterval": 5000000000, "AccountsRebuildSynchronousMode": 1, "AgreementIncomingBundlesQueueLength": 15, @@ -103,12 +103,17 @@ "ProposalAssemblyTime": 500000000, "PublicAddress": "", "ReconnectTime": 60000000000, + "RenaissanceCatchupLnProvenWeight": 0, + "RenaissanceCatchupProto": "", + "RenaissanceCatchupRound": 0, + "RenaissanceCatchupVotersCommitment": "", "ReservedFDs": 256, "RestConnectionsHardLimit": 2048, "RestConnectionsSoftLimit": 1024, "RestReadTimeoutSeconds": 15, "RestWriteTimeoutSeconds": 120, "RunHosted": false, + "StateproofCatchupDir": "", "StateproofDir": "", "StorageEngine": "sqlite", "SuggestedFeeBlockHistory": 3, diff --git a/ledger/apply/stateproof_test.go b/ledger/apply/stateproof_test.go index 155a4eef5d..efc877d97c 100644 --- a/ledger/apply/stateproof_test.go +++ b/ledger/apply/stateproof_test.go @@ -106,7 +106,7 @@ func TestApplyStateProofV34(t *testing.T) { stateProofTx.StateProofType = protocol.StateProofBasic // stateproof txn doesn't confirm the next state proof round. expected is in the past validate = true - stateProofTx.Message.LastAttestedRound = uint64(16) + stateProofTx.Message.LastAttestedRound = 16 applier.SetStateProofNextRound(8) err = StateProof(stateProofTx, atRound, applier, validate) a.ErrorIs(err, ErrExpectedDifferentStateProofRound) @@ -114,7 +114,7 @@ func TestApplyStateProofV34(t *testing.T) { // stateproof txn doesn't confirm the next state proof round. expected is in the future validate = true - stateProofTx.Message.LastAttestedRound = uint64(16) + stateProofTx.Message.LastAttestedRound = 16 applier.SetStateProofNextRound(32) err = StateProof(stateProofTx, atRound, applier, validate) a.ErrorIs(err, ErrExpectedDifferentStateProofRound) @@ -152,7 +152,7 @@ func TestApplyStateProofV34(t *testing.T) { spHdr.Round = 15 blocks[spHdr.Round] = spHdr - stateProofTx.Message.LastAttestedRound = uint64(spHdr.Round) + stateProofTx.Message.LastAttestedRound = spHdr.Round applier.SetStateProofNextRound(15) blockErr[13] = noBlockErr err = StateProof(stateProofTx, atRound, applier, validate) @@ -179,7 +179,7 @@ func TestApplyStateProofV34(t *testing.T) { atRoundBlock.CurrentProtocol = version blocks[atRound] = atRoundBlock - stateProofTx.Message.LastAttestedRound = 2 * config.Consensus[version].StateProofInterval + stateProofTx.Message.LastAttestedRound = 2 * basics.Round(config.Consensus[version].StateProofInterval) stateProofTx.StateProof.SignedWeight = 100 applier.SetStateProofNextRound(basics.Round(2 * config.Consensus[version].StateProofInterval)) @@ -220,7 +220,7 @@ func TestApplyStateProof(t *testing.T) { stateProofTx.StateProofType = protocol.StateProofBasic // stateproof txn doesn't confirm the next state proof round. expected is in the past validate = true - stateProofTx.Message.LastAttestedRound = uint64(16) + stateProofTx.Message.LastAttestedRound = 16 applier.SetStateProofNextRound(8) err = StateProof(stateProofTx, atRound, applier, validate) a.ErrorIs(err, ErrExpectedDifferentStateProofRound) @@ -228,7 +228,7 @@ func TestApplyStateProof(t *testing.T) { // stateproof txn doesn't confirm the next state proof round. expected is in the future validate = true - stateProofTx.Message.LastAttestedRound = uint64(16) + stateProofTx.Message.LastAttestedRound = 16 applier.SetStateProofNextRound(32) err = StateProof(stateProofTx, atRound, applier, validate) a.ErrorIs(err, ErrExpectedDifferentStateProofRound) diff --git a/ledger/eval/cow_test.go b/ledger/eval/cow_test.go index 225b037997..c1d2d03d25 100644 --- a/ledger/eval/cow_test.go +++ b/ledger/eval/cow_test.go @@ -307,20 +307,20 @@ func TestCowStateProof(t *testing.T) { c0.SetStateProofNextRound(firstStateproof) stateproofTxn := transactions.StateProofTxnFields{ StateProofType: protocol.StateProofBasic, - Message: stateproofmsg.Message{LastAttestedRound: uint64(firstStateproof) + version.StateProofInterval}, + Message: stateproofmsg.Message{LastAttestedRound: firstStateproof + basics.Round(version.StateProofInterval)}, } // can not apply state proof for 3*version.StateProofInterval when we expect 2*version.StateProofInterval err := apply.StateProof(stateproofTxn, firstStateproof+1, c0, false) a.ErrorIs(err, apply.ErrExpectedDifferentStateProofRound) - stateproofTxn.Message.LastAttestedRound = uint64(firstStateproof) + stateproofTxn.Message.LastAttestedRound = firstStateproof err = apply.StateProof(stateproofTxn, firstStateproof+1, c0, false) a.NoError(err) a.Equal(3*basics.Round(version.StateProofInterval), c0.GetStateProofNextRound()) // try to apply the next stateproof 3*version.StateProofInterval - stateproofTxn.Message.LastAttestedRound = 3 * version.StateProofInterval + stateproofTxn.Message.LastAttestedRound = basics.Round(3 * version.StateProofInterval) err = apply.StateProof(stateproofTxn, firstStateproof+1, c0, false) a.NoError(err) a.Equal(4*basics.Round(version.StateProofInterval), c0.GetStateProofNextRound()) diff --git a/node/follower_node.go b/node/follower_node.go index b7fb81065b..68dde144fd 100644 --- a/node/follower_node.go +++ b/node/follower_node.go @@ -20,6 +20,7 @@ package node import ( "context" "fmt" + "path/filepath" "time" "github.com/algorand/go-deadlock" @@ -40,6 +41,7 @@ import ( "github.com/algorand/go-algorand/network" "github.com/algorand/go-algorand/protocol" "github.com/algorand/go-algorand/rpcs" + "github.com/algorand/go-algorand/util/db" "github.com/algorand/go-algorand/util/execpool" ) @@ -126,10 +128,18 @@ func MakeFollower(log logging.Logger, rootDir string, cfg config.Local, phoneboo node, } + spCatchupFilename := filepath.Join(node.genesisDirs.StateproofCatchupGenesisDir, config.StateProofCatchupFilename) + spCatchupDB, err := db.MakeAccessor(spCatchupFilename, false, false) + if err != nil { + log.Errorf("Cannot create state-proof catchup DB (%s): %v", spCatchupFilename, err) + return nil, err + } + node.ledger.RegisterBlockListeners(blockListeners) node.blockService = rpcs.MakeBlockService(node.log, cfg, node.ledger, p2pNode, node.genesisID) node.catchupBlockAuth = blockAuthenticatorImpl{Ledger: node.ledger, AsyncVoteVerifier: agreement.MakeAsyncVoteVerifier(node.lowPriorityCryptoVerificationPool)} - node.catchupService = catchup.MakeService(node.log, node.config, p2pNode, node.ledger, node.catchupBlockAuth, make(chan catchup.PendingUnmatchedCertificate), node.lowPriorityCryptoVerificationPool) + node.catchupService = catchup.MakeService(node.log, node.config, p2pNode, node.ledger, node.catchupBlockAuth, make(chan catchup.PendingUnmatchedCertificate), node.lowPriorityCryptoVerificationPool, &spCatchupDB) + node.catchupService.SetRenaissanceFromConfig(cfg) // Initialize sync round to the latest db round + 1 so that nothing falls out of the cache on Start err = node.SetSyncRound(uint64(node.Ledger().LatestTrackerCommitted() + 1)) diff --git a/node/node.go b/node/node.go index 4c18ad1d51..2f2aa68d96 100644 --- a/node/node.go +++ b/node/node.go @@ -291,8 +291,16 @@ func MakeFull(log logging.Logger, rootDir string, cfg config.Local, phonebookAdd return nil, err } + spCatchupFilename := filepath.Join(node.genesisDirs.StateproofCatchupGenesisDir, config.StateProofCatchupFilename) + spCatchupDB, err := db.MakeAccessor(spCatchupFilename, false, false) + if err != nil { + log.Errorf("Cannot create state-proof catchup DB (%s): %v", spCatchupFilename, err) + return nil, err + } + node.catchupBlockAuth = blockAuthenticatorImpl{Ledger: node.ledger, AsyncVoteVerifier: agreement.MakeAsyncVoteVerifier(node.lowPriorityCryptoVerificationPool)} - node.catchupService = catchup.MakeService(node.log, node.config, p2pNode, node.ledger, node.catchupBlockAuth, agreementLedger.UnmatchedPendingCertificates, node.lowPriorityCryptoVerificationPool) + node.catchupService = catchup.MakeService(node.log, node.config, p2pNode, node.ledger, node.catchupBlockAuth, agreementLedger.UnmatchedPendingCertificates, node.lowPriorityCryptoVerificationPool, &spCatchupDB) + node.catchupService.SetRenaissanceFromConfig(cfg) node.txPoolSyncerService = rpcs.MakeTxSyncer(node.transactionPool, node.net, node.txHandler.SolicitedTxHandler(), time.Duration(cfg.TxSyncIntervalSeconds)*time.Second, time.Duration(cfg.TxSyncTimeoutSeconds)*time.Second, cfg.TxSyncServeResponseSize) registry, err := ensureParticipationDB(node.genesisDirs.ColdGenesisDir, node.log) diff --git a/rpcs/blockService.go b/rpcs/blockService.go index b185f224e5..7e791c0ad0 100644 --- a/rpcs/blockService.go +++ b/rpcs/blockService.go @@ -37,12 +37,16 @@ import ( "github.com/algorand/go-algorand/agreement" "github.com/algorand/go-algorand/config" "github.com/algorand/go-algorand/crypto" + cryptostateproof "github.com/algorand/go-algorand/crypto/stateproof" "github.com/algorand/go-algorand/data/basics" "github.com/algorand/go-algorand/data/bookkeeping" + "github.com/algorand/go-algorand/data/stateproofmsg" + "github.com/algorand/go-algorand/data/transactions" "github.com/algorand/go-algorand/ledger/ledgercore" "github.com/algorand/go-algorand/logging" "github.com/algorand/go-algorand/network" "github.com/algorand/go-algorand/protocol" + "github.com/algorand/go-algorand/stateproof" "github.com/algorand/go-algorand/util/metrics" ) @@ -53,11 +57,18 @@ const blockResponseMissingBlockCacheControl = "public, max-age=1, must-revalidat const blockResponseRetryAfter = "3" // retry after 3 seconds const blockServerMaxBodyLength = 512 // we don't really pass meaningful content here, so 512 bytes should be a safe limit const blockServerCatchupRequestBufferSize = 10 +const stateProofMaxCount = 128 + +// StateProofResponseContentType is the HTTP Content-Type header for a raw state proof transaction +const StateProofResponseContentType = "application/x-algorand-stateproof-v1" // BlockServiceBlockPath is the path to register BlockService as a handler for when using gorilla/mux // e.g. .HandleFunc(BlockServiceBlockPath, ls.ServeBlockPath) const BlockServiceBlockPath = "/v{version:[0-9.]+}/{genesisID}/block/{round:[0-9a-z]+}" +// BlockServiceStateProofPath is the path to register BlockService's ServeStateProofPath handler +const BlockServiceStateProofPath = "/v{version:[0-9.]+}/{genesisID}/stateproof/type{type:[0-9.]+}/{round:[0-9a-z]+}" + // Constant strings used as keys for topics const ( RoundKey = "roundKey" // Block round-number topic-key in the request @@ -87,6 +98,10 @@ var httpBlockMessagesDroppedCounter = metrics.MakeCounter( // LedgerForBlockService describes the Ledger methods used by BlockService. type LedgerForBlockService interface { EncodedBlockCert(rnd basics.Round) (blk []byte, cert []byte, err error) + BlockHdr(rnd basics.Round) (bookkeeping.BlockHeader, error) + Block(rnd basics.Round) (bookkeeping.Block, error) + Latest() basics.Round + AddressTxns(id basics.Address, r basics.Round) ([]transactions.SignedTxnWithAD, error) } // BlockService represents the Block RPC API @@ -108,12 +123,17 @@ type BlockService struct { memoryCap uint64 } -// EncodedBlockCert defines how GetBlockBytes encodes a block and its certificate +// EncodedBlockCert defines how GetBlockBytes and GetBlockStateProofBytes +// encodes a block and its certificate or light header proof. It is +// compatible with encoding of PreEncodedBlockCert, but currently we use +// two different structs, because we don't store pre-msgpack'ed light +// header proofs. type EncodedBlockCert struct { - _struct struct{} `codec:""` + _struct struct{} `codec:",omitempty,omitemptyarray"` - Block bookkeeping.Block `codec:"block"` - Certificate agreement.Certificate `codec:"cert"` + Block bookkeeping.Block `codec:"block,omitempty"` + Certificate agreement.Certificate `codec:"cert,omitempty"` + LightBlockHeaderProof []byte `codec:"proof,omitempty"` } // PreEncodedBlockCert defines how GetBlockBytes encodes a block and its certificate, @@ -159,6 +179,7 @@ type HTTPRegistrar interface { // RegisterHandlers registers the request handlers for BlockService's paths with the registrar. func (bs *BlockService) RegisterHandlers(registrar HTTPRegistrar) { registrar.RegisterHTTPHandlerFunc(BlockServiceBlockPath, bs.ServeBlockPath) + registrar.RegisterHTTPHandlerFunc(BlockServiceStateProofPath, bs.ServeStateProofPath) } // Start listening to catchup requests over ws @@ -185,8 +206,153 @@ func (bs *BlockService) Stop() { bs.closeWaitGroup.Wait() } +// OneStateProof is used to encode one state proof in the response +// to a state proof fetch request. +type OneStateProof struct { + _struct struct{} `codec:",omitempty,omitemptyarray"` + + StateProof cryptostateproof.StateProof `codec:"sp"` + Message stateproofmsg.Message `codec:"spmsg"` +} + +// StateProofResponse is used to encode the response to a state proof +// fetch request, consisting of one or more state proofs. +type StateProofResponse struct { + _struct struct{} `codec:",omitempty,omitemptyarray"` + + Proofs []OneStateProof `codec:"p,allocbound=stateProofMaxCount"` +} + +// ServeStateProofPath returns state proofs, starting with the specified round. +// It expects to be invoked via: +// +// /v{version}/{genesisID}/stateproof/type{type}/{round} +func (bs *BlockService) ServeStateProofPath(response http.ResponseWriter, request *http.Request) { + pathVars := mux.Vars(request) + versionStr := pathVars["version"] + roundStr := pathVars["round"] + genesisID := pathVars["genesisID"] + typeStr := pathVars["type"] + if versionStr != "1" { + bs.log.Debug("http stateproof bad version", versionStr) + response.WriteHeader(http.StatusBadRequest) + return + } + if genesisID != bs.genesisID { + bs.log.Debugf("http stateproof bad genesisID mine=%#v theirs=%#v", bs.genesisID, genesisID) + response.WriteHeader(http.StatusBadRequest) + return + } + if typeStr != "0" { // StateProofBasic + bs.log.Debugf("http stateproof bad type %s", typeStr) + response.WriteHeader(http.StatusBadRequest) + return + } + uround, err := strconv.ParseUint(roundStr, 36, 64) + if err != nil { + bs.log.Debug("http stateproof round parse fail", roundStr, err) + response.WriteHeader(http.StatusBadRequest) + return + } + round := basics.Round(uround) + + var res StateProofResponse + + latestRound := bs.ledger.Latest() + ctx := request.Context() + hdr, err := bs.ledger.BlockHdr(round) + if err != nil { + bs.log.Debug("http stateproof cannot get blockhdr", round, err) + switch err.(type) { + case ledgercore.ErrNoEntry: + // Send a 404 response, since res.Proofs is empty. + goto done + default: + response.WriteHeader(http.StatusInternalServerError) + return + } + } + + if config.Consensus[hdr.CurrentProtocol].StateProofInterval == 0 { + bs.log.Debug("http stateproof not enabled", round) + response.WriteHeader(http.StatusBadRequest) + return + } + + // As an optimization to prevent expensive searches for state + // proofs that don't exist yet, don't bother searching if we + // are looking for a state proof for a round that's within + // StateProofInterval of latest. + if round+basics.Round(config.Consensus[hdr.CurrentProtocol].StateProofInterval) >= latestRound { + // Send a 404 response, since res.Proofs is empty. + goto done + } + + for i := round + 1; i <= latestRound; i++ { + select { + case <-ctx.Done(): + return + default: + } + + if len(res.Proofs) >= stateProofMaxCount { + break + } + + txns, err := bs.ledger.AddressTxns(transactions.StateProofSender, i) + if err != nil { + bs.log.Debug("http stateproof address txns error", err) + response.WriteHeader(http.StatusInternalServerError) + return + } + for _, txn := range txns { + if txn.Txn.Type != protocol.StateProofTx { + continue + } + + if txn.Txn.StateProofTxnFields.Message.FirstAttestedRound <= round && round <= txn.Txn.StateProofTxnFields.Message.LastAttestedRound { + res.Proofs = append(res.Proofs, OneStateProof{ + StateProof: txn.Txn.StateProofTxnFields.StateProof, + Message: txn.Txn.StateProofTxnFields.Message, + }) + + // Keep looking for more state proofs, since the caller will + // likely want a sequence of them until the latest round. + round = round + basics.Round(config.Consensus[hdr.CurrentProtocol].StateProofInterval) + } + } + } + +done: + if len(res.Proofs) == 0 { + ok := bs.redirectRequest(response, request, bs.formatStateProofQuery(uint64(round))) + if !ok { + response.Header().Set("Cache-Control", blockResponseMissingBlockCacheControl) + response.WriteHeader(http.StatusNotFound) + } + } else { + encodedResponse := protocol.Encode(&res) + response.Header().Set("Content-Type", StateProofResponseContentType) + response.Header().Set("Content-Length", strconv.Itoa(len(encodedResponse))) + response.Header().Set("Cache-Control", blockResponseHasBlockCacheControl) + response.WriteHeader(http.StatusOK) + _, err = response.Write(encodedResponse) + if err != nil { + bs.log.Warn("http stateproof write failed ", err) + } + } +} + // ServeBlockPath returns blocks -// Either /v{version}/{genesisID}/block/{round} or ?b={round}&v={version} +// It expects to be invoked via several possible paths: +// +// /v{version}/{genesisID}/block/{round} +// ?b={round}&v={version} +// +// It optionally takes a ?stateproof={n} argument, where n is the type of +// state proof for which we should return a light block header proof. In +// the absence of this argument, we will return an agreement certificate. +// // Uses gorilla/mux for path argument parsing. func (bs *BlockService) ServeBlockPath(response http.ResponseWriter, request *http.Request) { pathVars := mux.Vars(request) @@ -211,15 +377,17 @@ func (bs *BlockService) ServeBlockPath(response http.ResponseWriter, request *ht response.WriteHeader(http.StatusBadRequest) return } + + request.Body = http.MaxBytesReader(response, request.Body, blockServerMaxBodyLength) + err := request.ParseForm() + if err != nil { + bs.log.Debug("http block parse form err", err) + response.WriteHeader(http.StatusBadRequest) + return + } + if (!hasVersionStr) || (!hasRoundStr) { // try query arg ?b={round} - request.Body = http.MaxBytesReader(response, request.Body, blockServerMaxBodyLength) - err := request.ParseForm() - if err != nil { - bs.log.Debug("http block parse form err", err) - response.WriteHeader(http.StatusBadRequest) - return - } roundStrs, ok := request.Form["b"] if !ok || len(roundStrs) != 1 { bs.log.Debug("http block bad block id form arg") @@ -248,12 +416,31 @@ func (bs *BlockService) ServeBlockPath(response http.ResponseWriter, request *ht response.WriteHeader(http.StatusBadRequest) return } - encodedBlockCert, err := bs.rawBlockBytes(basics.Round(round)) + + sendStateProof := false + stateProof, hasStateProof := request.Form["stateproof"] + if hasStateProof { + if len(stateProof) != 1 || stateProof[0] != "0" { + bs.log.Debugf("http block stateproof version %v unsupported", stateProof) + response.WriteHeader(http.StatusBadRequest) + return + } + + sendStateProof = true + } + + var encodedBlock []byte + if sendStateProof { + encodedBlock, err = bs.rawBlockStateProofBytes(basics.Round(round)) + } else { + encodedBlock, err = bs.rawBlockBytes(basics.Round(round)) + } + if err != nil { switch err.(type) { case ledgercore.ErrNoEntry: // entry cound not be found. - ok := bs.redirectRequest(round, response, request) + ok := bs.redirectRequest(response, request, bs.formatBlockQuery(round, sendStateProof)) if !ok { response.Header().Set("Cache-Control", blockResponseMissingBlockCacheControl) response.WriteHeader(http.StatusNotFound) @@ -261,7 +448,7 @@ func (bs *BlockService) ServeBlockPath(response http.ResponseWriter, request *ht return case errMemoryAtCapacity: // memory used by HTTP block requests is over the cap - ok := bs.redirectRequest(round, response, request) + ok := bs.redirectRequest(response, request, bs.formatBlockQuery(round, sendStateProof)) if !ok { response.Header().Set("Retry-After", blockResponseRetryAfter) response.WriteHeader(http.StatusServiceUnavailable) @@ -278,16 +465,16 @@ func (bs *BlockService) ServeBlockPath(response http.ResponseWriter, request *ht } response.Header().Set("Content-Type", BlockResponseContentType) - response.Header().Set("Content-Length", strconv.Itoa(len(encodedBlockCert))) + response.Header().Set("Content-Length", strconv.Itoa(len(encodedBlock))) response.Header().Set("Cache-Control", blockResponseHasBlockCacheControl) response.WriteHeader(http.StatusOK) - _, err = response.Write(encodedBlockCert) + _, err = response.Write(encodedBlock) if err != nil { bs.log.Warn("http block write failed ", err) } bs.mu.Lock() defer bs.mu.Unlock() - bs.memoryUsed = bs.memoryUsed - uint64(len(encodedBlockCert)) + bs.memoryUsed = bs.memoryUsed - uint64(len(encodedBlock)) } func (bs *BlockService) processIncomingMessage(msg network.IncomingMessage) (n network.OutgoingMessage) { @@ -392,7 +579,7 @@ func (bs *BlockService) handleCatchupReq(ctx context.Context, reqMsg network.Inc // redirectRequest redirects the request to the next round robin fallback endpoing if available, otherwise, // if EnableBlockServiceFallbackToArchiver is enabled, redirects to a random archiver. -func (bs *BlockService) redirectRequest(round uint64, response http.ResponseWriter, request *http.Request) (ok bool) { +func (bs *BlockService) redirectRequest(response http.ResponseWriter, request *http.Request, pathsuffix string) (ok bool) { peerAddress := bs.getNextCustomFallbackEndpoint() if peerAddress == "" && bs.enableArchiverFallback { peerAddress = bs.getRandomArchiver() @@ -406,12 +593,25 @@ func (bs *BlockService) redirectRequest(round uint64, response http.ResponseWrit bs.log.Debugf("redirectRequest: %s", err.Error()) return false } - parsedURL.Path = strings.Replace(FormatBlockQuery(round, parsedURL.Path, bs.net), "{genesisID}", bs.genesisID, 1) + parsedURL.Path = path.Join(parsedURL.Path, pathsuffix) http.Redirect(response, request, parsedURL.String(), http.StatusTemporaryRedirect) bs.log.Debugf("redirectRequest: redirected block request to %s", parsedURL.String()) return true } +func (bs *BlockService) formatBlockQuery(round uint64, sendStateProof bool) string { + stateProofArg := "" + if sendStateProof { + stateProofArg = "?stateproof=0" + } + + return fmt.Sprintf("/v1/%s/block/%s%s", bs.genesisID, strconv.FormatUint(uint64(round), 36), stateProofArg) +} + +func (bs *BlockService) formatStateProofQuery(round uint64) string { + return fmt.Sprintf("/v1/%s/stateproof/type0/%s", bs.genesisID, strconv.FormatUint(uint64(round), 36)) +} + // getNextCustomFallbackEndpoint returns the next custorm fallback endpoint in RR ordering func (bs *BlockService) getNextCustomFallbackEndpoint() (endpointAddress string) { if len(bs.fallbackEndpoints.endpoints) == 0 { @@ -464,6 +664,29 @@ func (bs *BlockService) rawBlockBytes(round basics.Round) ([]byte, error) { return data, err } +// rawBlockStateProofBytes returns the block and light header proof for a given round, while taking the lock +// to ensure the block service is currently active. +func (bs *BlockService) rawBlockStateProofBytes(round basics.Round) ([]byte, error) { + bs.mu.Lock() + defer bs.mu.Unlock() + select { + case _, ok := <-bs.stop: + if !ok { + // service is closed. + return nil, errBlockServiceClosed + } + default: + } + if bs.memoryUsed > bs.memoryCap { + return nil, errMemoryAtCapacity{used: bs.memoryUsed, capacity: bs.memoryCap} + } + data, err := RawBlockStateProofBytes(bs.ledger, round) + if err == nil { + bs.memoryUsed = bs.memoryUsed + uint64(len(data)) + } + return data, err +} + func topicBlockBytes(log logging.Logger, dataLedger LedgerForBlockService, round basics.Round, requestType string) (network.Topics, uint64) { blk, cert, err := dataLedger.EncodedBlockCert(round) if err != nil { @@ -506,11 +729,55 @@ func RawBlockBytes(l LedgerForBlockService, round basics.Round) ([]byte, error) }), nil } +// RawBlockStateProofBytes return the msgpack bytes for a block and light header proof +func RawBlockStateProofBytes(l LedgerForBlockService, round basics.Round) ([]byte, error) { + blk, err := l.Block(round) + if err != nil { + return nil, err + } + + stateProofInterval := basics.Round(config.Consensus[blk.CurrentProtocol].StateProofInterval) + if stateProofInterval == 0 { + return nil, fmt.Errorf("state proofs not supported in block %d", round) + } + + lastAttestedRound := round.RoundUpToMultipleOf(stateProofInterval) + firstAttestedRound := lastAttestedRound - stateProofInterval + 1 + + latest := l.Latest() + if lastAttestedRound > latest { + return nil, fmt.Errorf("light block header proof not available for block %d yet, latest is %d", lastAttestedRound, latest) + } + + lightHeaders, err := stateproof.FetchLightHeaders(l, uint64(stateProofInterval), lastAttestedRound) + if err != nil { + return nil, err + } + + blockIndex := uint64(round - firstAttestedRound) + leafproof, err := stateproof.GenerateProofOfLightBlockHeaders(uint64(stateProofInterval), lightHeaders, blockIndex) + if err != nil { + return nil, err + } + + r := EncodedBlockCert{ + Block: blk, + LightBlockHeaderProof: leafproof.GetConcatenatedProof(), + } + + return protocol.Encode(&r), nil +} + // FormatBlockQuery formats a block request query for the given network and round number func FormatBlockQuery(round uint64, parsedURL string, net network.GossipNode) string { return net.SubstituteGenesisID(path.Join(parsedURL, "/v1/{genesisID}/block/"+strconv.FormatUint(uint64(round), 36))) } +// FormatStateProofQuery formats a state proof request for the given network, proof type, and round number +func FormatStateProofQuery(round uint64, proofType protocol.StateProofType, parsedURL string, net network.GossipNode) string { + return net.SubstituteGenesisID(path.Join(parsedURL, "/v1/{genesisID}/stateproof/type"+strconv.FormatUint(uint64(proofType), 10)+"/"+strconv.FormatUint(uint64(round), 36))) +} + func makeFallbackEndpoints(log logging.Logger, customFallbackEndpoints string) (fe fallbackEndpoints) { if customFallbackEndpoints == "" { return diff --git a/rpcs/ledgerService.go b/rpcs/ledgerService.go index 8abf87e3ba..3e21bdfa12 100644 --- a/rpcs/ledgerService.go +++ b/rpcs/ledgerService.go @@ -101,7 +101,7 @@ func (ls *LedgerService) Stop() { } } -// ServerHTTP returns ledgers for a particular round +// ServeHTTP returns ledgers for a particular round // Either /v{version}/{genesisID}/ledger/{round} or ?r={round}&v={version} // Uses gorilla/mux for path argument parsing. func (ls *LedgerService) ServeHTTP(response http.ResponseWriter, request *http.Request) { diff --git a/rpcs/msgp_gen.go b/rpcs/msgp_gen.go index 5f8af433f7..2ece973295 100644 --- a/rpcs/msgp_gen.go +++ b/rpcs/msgp_gen.go @@ -6,7 +6,9 @@ import ( "github.com/algorand/msgp/msgp" "github.com/algorand/go-algorand/agreement" + cryptostateproof "github.com/algorand/go-algorand/crypto/stateproof" "github.com/algorand/go-algorand/data/bookkeeping" + "github.com/algorand/go-algorand/data/stateproofmsg" ) // The following msgp objects are implemented in this file: @@ -19,17 +21,62 @@ import ( // |-----> (*) MsgIsZero // |-----> EncodedBlockCertMaxSize() // +// OneStateProof +// |-----> (*) MarshalMsg +// |-----> (*) CanMarshalMsg +// |-----> (*) UnmarshalMsg +// |-----> (*) CanUnmarshalMsg +// |-----> (*) Msgsize +// |-----> (*) MsgIsZero +// |-----> OneStateProofMaxSize() +// +// StateProofResponse +// |-----> (*) MarshalMsg +// |-----> (*) CanMarshalMsg +// |-----> (*) UnmarshalMsg +// |-----> (*) CanUnmarshalMsg +// |-----> (*) Msgsize +// |-----> (*) MsgIsZero +// |-----> StateProofResponseMaxSize() +// // MarshalMsg implements msgp.Marshaler func (z *EncodedBlockCert) MarshalMsg(b []byte) (o []byte) { o = msgp.Require(b, z.Msgsize()) - // map header, size 2 - // string "block" - o = append(o, 0x82, 0xa5, 0x62, 0x6c, 0x6f, 0x63, 0x6b) - o = (*z).Block.MarshalMsg(o) - // string "cert" - o = append(o, 0xa4, 0x63, 0x65, 0x72, 0x74) - o = (*z).Certificate.MarshalMsg(o) + // omitempty: check for empty values + zb0001Len := uint32(3) + var zb0001Mask uint8 /* 4 bits */ + if (*z).Block.MsgIsZero() { + zb0001Len-- + zb0001Mask |= 0x2 + } + if (*z).Certificate.MsgIsZero() { + zb0001Len-- + zb0001Mask |= 0x4 + } + if len((*z).LightBlockHeaderProof) == 0 { + zb0001Len-- + zb0001Mask |= 0x8 + } + // variable map header, size zb0001Len + o = append(o, 0x80|uint8(zb0001Len)) + if zb0001Len != 0 { + if (zb0001Mask & 0x2) == 0 { // if not empty + // string "block" + o = append(o, 0xa5, 0x62, 0x6c, 0x6f, 0x63, 0x6b) + o = (*z).Block.MarshalMsg(o) + } + if (zb0001Mask & 0x4) == 0 { // if not empty + // string "cert" + o = append(o, 0xa4, 0x63, 0x65, 0x72, 0x74) + o = (*z).Certificate.MarshalMsg(o) + } + if (zb0001Mask & 0x8) == 0 { // if not empty + // string "proof" + o = append(o, 0xa5, 0x70, 0x72, 0x6f, 0x6f, 0x66) + o = msgp.AppendBytes(o, (*z).LightBlockHeaderProof) + } + } return } @@ -67,6 +114,14 @@ func (z *EncodedBlockCert) UnmarshalMsg(bts []byte) (o []byte, err error) { return } } + if zb0001 > 0 { + zb0001-- + (*z).LightBlockHeaderProof, bts, err = msgp.ReadBytesBytes(bts, (*z).LightBlockHeaderProof) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "LightBlockHeaderProof") + return + } + } if zb0001 > 0 { err = msgp.ErrTooManyArrayFields(zb0001) if err != nil { @@ -102,6 +157,12 @@ func (z *EncodedBlockCert) UnmarshalMsg(bts []byte) (o []byte, err error) { err = msgp.WrapError(err, "Certificate") return } + case "proof": + (*z).LightBlockHeaderProof, bts, err = msgp.ReadBytesBytes(bts, (*z).LightBlockHeaderProof) + if err != nil { + err = msgp.WrapError(err, "LightBlockHeaderProof") + return + } default: err = msgp.ErrNoField(string(field)) if err != nil { @@ -122,17 +183,469 @@ func (_ *EncodedBlockCert) CanUnmarshalMsg(z interface{}) bool { // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message func (z *EncodedBlockCert) Msgsize() (s int) { - s = 1 + 6 + (*z).Block.Msgsize() + 5 + (*z).Certificate.Msgsize() + s = 1 + 6 + (*z).Block.Msgsize() + 5 + (*z).Certificate.Msgsize() + 6 + msgp.BytesPrefixSize + len((*z).LightBlockHeaderProof) return } // MsgIsZero returns whether this is a zero value func (z *EncodedBlockCert) MsgIsZero() bool { - return ((*z).Block.MsgIsZero()) && ((*z).Certificate.MsgIsZero()) + return ((*z).Block.MsgIsZero()) && ((*z).Certificate.MsgIsZero()) && (len((*z).LightBlockHeaderProof) == 0) } // MaxSize returns a maximum valid message size for this message type func EncodedBlockCertMaxSize() (s int) { - s = 1 + 6 + bookkeeping.BlockMaxSize() + 5 + agreement.CertificateMaxSize() + s = 1 + 6 + bookkeeping.BlockMaxSize() + 5 + agreement.CertificateMaxSize() + 6 + panic("Unable to determine max size: Byteslice type z.LightBlockHeaderProof is unbounded") + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *OneStateProof) MarshalMsg(b []byte) (o []byte) { + o = msgp.Require(b, z.Msgsize()) + // omitempty: check for empty values + zb0001Len := uint32(2) + var zb0001Mask uint8 /* 3 bits */ + if (*z).StateProof.MsgIsZero() { + zb0001Len-- + zb0001Mask |= 0x2 + } + if (*z).Message.MsgIsZero() { + zb0001Len-- + zb0001Mask |= 0x4 + } + // variable map header, size zb0001Len + o = append(o, 0x80|uint8(zb0001Len)) + if zb0001Len != 0 { + if (zb0001Mask & 0x2) == 0 { // if not empty + // string "sp" + o = append(o, 0xa2, 0x73, 0x70) + o = (*z).StateProof.MarshalMsg(o) + } + if (zb0001Mask & 0x4) == 0 { // if not empty + // string "spmsg" + o = append(o, 0xa5, 0x73, 0x70, 0x6d, 0x73, 0x67) + o = (*z).Message.MarshalMsg(o) + } + } + return +} + +func (_ *OneStateProof) CanMarshalMsg(z interface{}) bool { + _, ok := (z).(*OneStateProof) + return ok +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *OneStateProof) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 int + var zb0002 bool + zb0001, zb0002, bts, err = msgp.ReadMapHeaderBytes(bts) + if _, ok := err.(msgp.TypeError); ok { + zb0001, zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + if zb0001 > 0 { + zb0001-- + bts, err = (*z).StateProof.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "StateProof") + return + } + } + if zb0001 > 0 { + zb0001-- + bts, err = (*z).Message.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "Message") + return + } + } + if zb0001 > 0 { + err = msgp.ErrTooManyArrayFields(zb0001) + if err != nil { + err = msgp.WrapError(err, "struct-from-array") + return + } + } + } else { + if err != nil { + err = msgp.WrapError(err) + return + } + if zb0002 { + (*z) = OneStateProof{} + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch string(field) { + case "sp": + bts, err = (*z).StateProof.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "StateProof") + return + } + case "spmsg": + bts, err = (*z).Message.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Message") + return + } + default: + err = msgp.ErrNoField(string(field)) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + } + o = bts + return +} + +func (_ *OneStateProof) CanUnmarshalMsg(z interface{}) bool { + _, ok := (z).(*OneStateProof) + return ok +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *OneStateProof) Msgsize() (s int) { + s = 1 + 3 + (*z).StateProof.Msgsize() + 6 + (*z).Message.Msgsize() + return +} + +// MsgIsZero returns whether this is a zero value +func (z *OneStateProof) MsgIsZero() bool { + return ((*z).StateProof.MsgIsZero()) && ((*z).Message.MsgIsZero()) +} + +// MaxSize returns a maximum valid message size for this message type +func OneStateProofMaxSize() (s int) { + s = 1 + 3 + cryptostateproof.StateProofMaxSize() + 6 + stateproofmsg.MessageMaxSize() + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *StateProofResponse) MarshalMsg(b []byte) (o []byte) { + o = msgp.Require(b, z.Msgsize()) + // omitempty: check for empty values + zb0002Len := uint32(1) + var zb0002Mask uint8 /* 2 bits */ + if len((*z).Proofs) == 0 { + zb0002Len-- + zb0002Mask |= 0x2 + } + // variable map header, size zb0002Len + o = append(o, 0x80|uint8(zb0002Len)) + if zb0002Len != 0 { + if (zb0002Mask & 0x2) == 0 { // if not empty + // string "p" + o = append(o, 0xa1, 0x70) + if (*z).Proofs == nil { + o = msgp.AppendNil(o) + } else { + o = msgp.AppendArrayHeader(o, uint32(len((*z).Proofs))) + } + for zb0001 := range (*z).Proofs { + // omitempty: check for empty values + zb0003Len := uint32(2) + var zb0003Mask uint8 /* 3 bits */ + if (*z).Proofs[zb0001].StateProof.MsgIsZero() { + zb0003Len-- + zb0003Mask |= 0x2 + } + if (*z).Proofs[zb0001].Message.MsgIsZero() { + zb0003Len-- + zb0003Mask |= 0x4 + } + // variable map header, size zb0003Len + o = append(o, 0x80|uint8(zb0003Len)) + if (zb0003Mask & 0x2) == 0 { // if not empty + // string "sp" + o = append(o, 0xa2, 0x73, 0x70) + o = (*z).Proofs[zb0001].StateProof.MarshalMsg(o) + } + if (zb0003Mask & 0x4) == 0 { // if not empty + // string "spmsg" + o = append(o, 0xa5, 0x73, 0x70, 0x6d, 0x73, 0x67) + o = (*z).Proofs[zb0001].Message.MarshalMsg(o) + } + } + } + } + return +} + +func (_ *StateProofResponse) CanMarshalMsg(z interface{}) bool { + _, ok := (z).(*StateProofResponse) + return ok +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *StateProofResponse) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0002 int + var zb0003 bool + zb0002, zb0003, bts, err = msgp.ReadMapHeaderBytes(bts) + if _, ok := err.(msgp.TypeError); ok { + zb0002, zb0003, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + if zb0002 > 0 { + zb0002-- + var zb0004 int + var zb0005 bool + zb0004, zb0005, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "Proofs") + return + } + if zb0004 > stateProofMaxCount { + err = msgp.ErrOverflow(uint64(zb0004), uint64(stateProofMaxCount)) + err = msgp.WrapError(err, "struct-from-array", "Proofs") + return + } + if zb0005 { + (*z).Proofs = nil + } else if (*z).Proofs != nil && cap((*z).Proofs) >= zb0004 { + (*z).Proofs = ((*z).Proofs)[:zb0004] + } else { + (*z).Proofs = make([]OneStateProof, zb0004) + } + for zb0001 := range (*z).Proofs { + var zb0006 int + var zb0007 bool + zb0006, zb0007, bts, err = msgp.ReadMapHeaderBytes(bts) + if _, ok := err.(msgp.TypeError); ok { + zb0006, zb0007, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "Proofs", zb0001) + return + } + if zb0006 > 0 { + zb0006-- + bts, err = (*z).Proofs[zb0001].StateProof.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "Proofs", zb0001, "struct-from-array", "StateProof") + return + } + } + if zb0006 > 0 { + zb0006-- + bts, err = (*z).Proofs[zb0001].Message.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "Proofs", zb0001, "struct-from-array", "Message") + return + } + } + if zb0006 > 0 { + err = msgp.ErrTooManyArrayFields(zb0006) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "Proofs", zb0001, "struct-from-array") + return + } + } + } else { + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "Proofs", zb0001) + return + } + if zb0007 { + (*z).Proofs[zb0001] = OneStateProof{} + } + for zb0006 > 0 { + zb0006-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "Proofs", zb0001) + return + } + switch string(field) { + case "sp": + bts, err = (*z).Proofs[zb0001].StateProof.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "Proofs", zb0001, "StateProof") + return + } + case "spmsg": + bts, err = (*z).Proofs[zb0001].Message.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "Proofs", zb0001, "Message") + return + } + default: + err = msgp.ErrNoField(string(field)) + if err != nil { + err = msgp.WrapError(err, "struct-from-array", "Proofs", zb0001) + return + } + } + } + } + } + } + if zb0002 > 0 { + err = msgp.ErrTooManyArrayFields(zb0002) + if err != nil { + err = msgp.WrapError(err, "struct-from-array") + return + } + } + } else { + if err != nil { + err = msgp.WrapError(err) + return + } + if zb0003 { + (*z) = StateProofResponse{} + } + for zb0002 > 0 { + zb0002-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch string(field) { + case "p": + var zb0008 int + var zb0009 bool + zb0008, zb0009, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Proofs") + return + } + if zb0008 > stateProofMaxCount { + err = msgp.ErrOverflow(uint64(zb0008), uint64(stateProofMaxCount)) + err = msgp.WrapError(err, "Proofs") + return + } + if zb0009 { + (*z).Proofs = nil + } else if (*z).Proofs != nil && cap((*z).Proofs) >= zb0008 { + (*z).Proofs = ((*z).Proofs)[:zb0008] + } else { + (*z).Proofs = make([]OneStateProof, zb0008) + } + for zb0001 := range (*z).Proofs { + var zb0010 int + var zb0011 bool + zb0010, zb0011, bts, err = msgp.ReadMapHeaderBytes(bts) + if _, ok := err.(msgp.TypeError); ok { + zb0010, zb0011, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Proofs", zb0001) + return + } + if zb0010 > 0 { + zb0010-- + bts, err = (*z).Proofs[zb0001].StateProof.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Proofs", zb0001, "struct-from-array", "StateProof") + return + } + } + if zb0010 > 0 { + zb0010-- + bts, err = (*z).Proofs[zb0001].Message.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Proofs", zb0001, "struct-from-array", "Message") + return + } + } + if zb0010 > 0 { + err = msgp.ErrTooManyArrayFields(zb0010) + if err != nil { + err = msgp.WrapError(err, "Proofs", zb0001, "struct-from-array") + return + } + } + } else { + if err != nil { + err = msgp.WrapError(err, "Proofs", zb0001) + return + } + if zb0011 { + (*z).Proofs[zb0001] = OneStateProof{} + } + for zb0010 > 0 { + zb0010-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err, "Proofs", zb0001) + return + } + switch string(field) { + case "sp": + bts, err = (*z).Proofs[zb0001].StateProof.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Proofs", zb0001, "StateProof") + return + } + case "spmsg": + bts, err = (*z).Proofs[zb0001].Message.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Proofs", zb0001, "Message") + return + } + default: + err = msgp.ErrNoField(string(field)) + if err != nil { + err = msgp.WrapError(err, "Proofs", zb0001) + return + } + } + } + } + } + default: + err = msgp.ErrNoField(string(field)) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + } + o = bts + return +} + +func (_ *StateProofResponse) CanUnmarshalMsg(z interface{}) bool { + _, ok := (z).(*StateProofResponse) + return ok +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *StateProofResponse) Msgsize() (s int) { + s = 1 + 2 + msgp.ArrayHeaderSize + for zb0001 := range (*z).Proofs { + s += 1 + 3 + (*z).Proofs[zb0001].StateProof.Msgsize() + 6 + (*z).Proofs[zb0001].Message.Msgsize() + } + return +} + +// MsgIsZero returns whether this is a zero value +func (z *StateProofResponse) MsgIsZero() bool { + return (len((*z).Proofs) == 0) +} + +// MaxSize returns a maximum valid message size for this message type +func StateProofResponseMaxSize() (s int) { + s = 1 + 2 + // Calculating size of slice: z.Proofs + s += msgp.ArrayHeaderSize + ((stateProofMaxCount) * (OneStateProofMaxSize())) return } diff --git a/rpcs/msgp_gen_test.go b/rpcs/msgp_gen_test.go index d88b73039b..30771c759e 100644 --- a/rpcs/msgp_gen_test.go +++ b/rpcs/msgp_gen_test.go @@ -73,3 +73,123 @@ func BenchmarkUnmarshalEncodedBlockCert(b *testing.B) { } } } + +func TestMarshalUnmarshalOneStateProof(t *testing.T) { + partitiontest.PartitionTest(t) + v := OneStateProof{} + bts := v.MarshalMsg(nil) + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func TestRandomizedEncodingOneStateProof(t *testing.T) { + protocol.RunEncodingTest(t, &OneStateProof{}) +} + +func BenchmarkMarshalMsgOneStateProof(b *testing.B) { + v := OneStateProof{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgOneStateProof(b *testing.B) { + v := OneStateProof{} + bts := make([]byte, 0, v.Msgsize()) + bts = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalOneStateProof(b *testing.B) { + v := OneStateProof{} + bts := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshalStateProofResponse(t *testing.T) { + partitiontest.PartitionTest(t) + v := StateProofResponse{} + bts := v.MarshalMsg(nil) + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func TestRandomizedEncodingStateProofResponse(t *testing.T) { + protocol.RunEncodingTest(t, &StateProofResponse{}) +} + +func BenchmarkMarshalMsgStateProofResponse(b *testing.B) { + v := StateProofResponse{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgStateProofResponse(b *testing.B) { + v := StateProofResponse{} + bts := make([]byte, 0, v.Msgsize()) + bts = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalStateProofResponse(b *testing.B) { + v := StateProofResponse{} + bts := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} diff --git a/stateproof/stateproofMessageGenerator.go b/stateproof/stateproofMessageGenerator.go index c51d767195..1437697103 100644 --- a/stateproof/stateproofMessageGenerator.go +++ b/stateproof/stateproofMessageGenerator.go @@ -59,7 +59,7 @@ func GenerateStateProofMessage(l BlockHeaderFetcher, round basics.Round) (statep } proto := config.Consensus[latestRoundHeader.CurrentProtocol] - votersRound := uint64(round.SubSaturate(basics.Round(proto.StateProofInterval))) + votersRound := round.SubSaturate(basics.Round(proto.StateProofInterval)) commitment, err := createHeaderCommitment(l, &proto, &latestRoundHeader) if err != nil { return stateproofmsg.Message{}, err @@ -75,7 +75,7 @@ func GenerateStateProofMessage(l BlockHeaderFetcher, round basics.Round) (statep VotersCommitment: latestRoundHeader.StateProofTracking[protocol.StateProofBasic].StateProofVotersCommitment, LnProvenWeight: lnProvenWeight, FirstAttestedRound: votersRound + 1, - LastAttestedRound: uint64(latestRoundHeader.Round), + LastAttestedRound: latestRoundHeader.Round, }, nil } diff --git a/stateproof/stateproofMessageGenerator_test.go b/stateproof/stateproofMessageGenerator_test.go index a990143b05..1c1e5d10d3 100644 --- a/stateproof/stateproofMessageGenerator_test.go +++ b/stateproof/stateproofMessageGenerator_test.go @@ -76,20 +76,20 @@ func TestStateProofMessage(t *testing.T) { if !lastMessage.MsgIsZero() { verifier := stateproof.MkVerifierWithLnProvenWeight(lastMessage.VotersCommitment, lastMessage.LnProvenWeight, proto.StateProofStrengthTarget) - err := verifier.Verify(tx.Txn.Message.LastAttestedRound, tx.Txn.Message.Hash(), &tx.Txn.StateProof) + err := verifier.Verify(uint64(tx.Txn.Message.LastAttestedRound), tx.Txn.Message.Hash(), &tx.Txn.StateProof) a.NoError(err) } // since a state proof txn was created, we update the header with the next state proof round // i.e network has accepted the state proof. - s.addBlock(basics.Round(tx.Txn.Message.LastAttestedRound + proto.StateProofInterval)) + s.addBlock(tx.Txn.Message.LastAttestedRound + basics.Round(proto.StateProofInterval)) lastMessage = tx.Txn.Message } } func verifySha256BlockHeadersCommitments(a *require.Assertions, message stateproofmsg.Message, blocks map[basics.Round]bookkeeping.BlockHeader) { blkHdrArr := make(lightBlockHeaders, message.LastAttestedRound-message.FirstAttestedRound+1) - for i := uint64(0); i < message.LastAttestedRound-message.FirstAttestedRound+1; i++ { - hdr := blocks[basics.Round(message.FirstAttestedRound+i)] + for i := uint64(0); i < uint64(message.LastAttestedRound-message.FirstAttestedRound+1); i++ { + hdr := blocks[message.FirstAttestedRound+basics.Round(i)] blkHdrArr[i] = hdr.ToLightBlockHeader() } @@ -217,7 +217,7 @@ func TestGenerateBlockProof(t *testing.T) { verifyLightBlockHeaderProof(&tx, &proto, headers, a) - s.addBlock(basics.Round(tx.Txn.Message.LastAttestedRound + proto.StateProofInterval)) + s.addBlock(tx.Txn.Message.LastAttestedRound + basics.Round(proto.StateProofInterval)) lastAttestedRound = basics.Round(tx.Txn.Message.LastAttestedRound) } } @@ -225,7 +225,7 @@ func TestGenerateBlockProof(t *testing.T) { func verifyLightBlockHeaderProof(tx *transactions.SignedTxn, proto *config.ConsensusParams, headers []bookkeeping.LightBlockHeader, a *require.Assertions) { // attempting to get block proof for every block in the interval for j := tx.Txn.Message.FirstAttestedRound; j < tx.Txn.Message.LastAttestedRound; j++ { - headerIndex := j - tx.Txn.Message.FirstAttestedRound + headerIndex := uint64(j - tx.Txn.Message.FirstAttestedRound) proof, err := GenerateProofOfLightBlockHeaders(proto.StateProofInterval, headers, headerIndex) a.NoError(err) a.NotNil(proof) diff --git a/stateproof/worker_test.go b/stateproof/worker_test.go index 68a19dcd21..0f9efbad55 100644 --- a/stateproof/worker_test.go +++ b/stateproof/worker_test.go @@ -742,7 +742,7 @@ func TestWorkerRestart(t *testing.T) { proto := config.Consensus[protocol.ConsensusCurrentVersion] s.advanceRoundsWithoutStateProof(t, 1) - lastRound := uint64(0) + lastRound := basics.Round(0) for i := 0; i < expectedStateProofs; i++ { s.advanceRoundsWithoutStateProof(t, proto.StateProofInterval/2-1) w.Stop() @@ -763,10 +763,10 @@ func TestWorkerRestart(t *testing.T) { // since a state proof txn was created, we update the header with the next state proof round // i.e network has accepted the state proof. - s.addBlock(basics.Round(tx.Txn.Message.LastAttestedRound + proto.StateProofInterval)) + s.addBlock(tx.Txn.Message.LastAttestedRound + basics.Round(proto.StateProofInterval)) lastRound = tx.Txn.Message.LastAttestedRound } - a.Equal(uint64(expectedStateProofs+1), lastRound/proto.StateProofInterval) + a.Equal(uint64(expectedStateProofs+1), uint64(lastRound)/proto.StateProofInterval) } func TestWorkerHandleSig(t *testing.T) { diff --git a/test/e2e-go/features/stateproofs/stateproofs_test.go b/test/e2e-go/features/stateproofs/stateproofs_test.go index bba0838c21..7c5ec26990 100644 --- a/test/e2e-go/features/stateproofs/stateproofs_test.go +++ b/test/e2e-go/features/stateproofs/stateproofs_test.go @@ -358,10 +358,10 @@ func TestStateProofMessageCommitmentVerification(t *testing.T) { t.Logf("found first stateproof, attesting to rounds %d - %d. Verifying.\n", stateProofMessage.FirstAttestedRound, stateProofMessage.LastAttestedRound) for rnd := stateProofMessage.FirstAttestedRound; rnd <= stateProofMessage.LastAttestedRound; rnd++ { - proofResp, singleLeafProof, err := fixture.LightBlockHeaderProof(rnd) + proofResp, singleLeafProof, err := fixture.LightBlockHeaderProof(uint64(rnd)) r.NoError(err) - blk, err := libgoalClient.BookkeepingBlock(rnd) + blk, err := libgoalClient.BookkeepingBlock(uint64(rnd)) r.NoError(err) lightBlockHeader := blk.ToLightBlockHeader() @@ -410,8 +410,8 @@ func getStateProofByLastRound(r *require.Assertions, fixture *fixtures.RestClien BlockHeadersCommitment: res.Message.BlockHeadersCommitment, VotersCommitment: res.Message.VotersCommitment, LnProvenWeight: res.Message.LnProvenWeight, - FirstAttestedRound: res.Message.FirstAttestedRound, - LastAttestedRound: res.Message.LastAttestedRound, + FirstAttestedRound: basics.Round(res.Message.FirstAttestedRound), + LastAttestedRound: basics.Round(res.Message.LastAttestedRound), } return stateProof, msg } @@ -1284,7 +1284,7 @@ func TestStateProofCheckTotalStake(t *testing.T) { stateProof, stateProofMsg := getStateProofByLastRound(r, &fixture, nextStateProofRound) - accountSnapshot := accountSnapshotAtRound[stateProofMsg.LastAttestedRound-consensusParams.StateProofInterval-consensusParams.StateProofVotersLookback] + accountSnapshot := accountSnapshotAtRound[uint64(stateProofMsg.LastAttestedRound)-consensusParams.StateProofInterval-consensusParams.StateProofVotersLookback] // once the state proof is accepted we want to make sure that the weight for _, v := range stateProof.Reveals { diff --git a/test/testdata/configs/config-v32.json b/test/testdata/configs/config-v32.json new file mode 100644 index 0000000000..47300a4332 --- /dev/null +++ b/test/testdata/configs/config-v32.json @@ -0,0 +1,139 @@ +{ + "Version": 32, + "AccountUpdatesStatsInterval": 5000000000, + "AccountsRebuildSynchronousMode": 1, + "AgreementIncomingBundlesQueueLength": 15, + "AgreementIncomingProposalsQueueLength": 50, + "AgreementIncomingVotesQueueLength": 20000, + "AnnounceParticipationKey": true, + "Archival": false, + "BaseLoggerDebugLevel": 4, + "BlockDBDir": "", + "BlockServiceCustomFallbackEndpoints": "", + "BlockServiceMemCap": 500000000, + "BroadcastConnectionsLimit": -1, + "CadaverDirectory": "", + "CadaverSizeTarget": 0, + "CatchpointDir": "", + "CatchpointFileHistoryLength": 365, + "CatchpointInterval": 10000, + "CatchpointTracking": 0, + "CatchupBlockDownloadRetryAttempts": 1000, + "CatchupBlockValidateMode": 0, + "CatchupFailurePeerRefreshRate": 10, + "CatchupGossipBlockFetchTimeoutSec": 4, + "CatchupHTTPBlockFetchTimeoutSec": 4, + "CatchupLedgerDownloadRetryAttempts": 50, + "CatchupParallelBlocks": 16, + "ColdDataDir": "", + "ConnectionsRateLimitingCount": 60, + "ConnectionsRateLimitingWindowSeconds": 1, + "CrashDBDir": "", + "DNSBootstrapID": ".algorand.network?backup=.algorand.net&dedup=.algorand-.(network|net)", + "DNSSecurityFlags": 1, + "DeadlockDetection": 0, + "DeadlockDetectionThreshold": 30, + "DisableAPIAuth": false, + "DisableLedgerLRUCache": false, + "DisableLocalhostConnectionRateLimit": true, + "DisableNetworking": false, + "DisableOutgoingConnectionThrottling": false, + "EnableAccountUpdatesStats": false, + "EnableAgreementReporting": false, + "EnableAgreementTimeMetrics": false, + "EnableAssembleStats": false, + "EnableBlockService": false, + "EnableBlockServiceFallbackToArchiver": true, + "EnableCatchupFromArchiveServers": false, + "EnableDeveloperAPI": false, + "EnableExperimentalAPI": false, + "EnableFollowMode": false, + "EnableGossipBlockService": true, + "EnableIncomingMessageFilter": false, + "EnableLedgerService": false, + "EnableMetricReporting": false, + "EnableOutgoingNetworkMessageFiltering": true, + "EnableP2P": false, + "EnablePingHandler": true, + "EnableProcessBlockStats": false, + "EnableProfiler": false, + "EnableRequestLogger": false, + "EnableRuntimeMetrics": false, + "EnableTopAccountsReporting": false, + "EnableTxBacklogRateLimiting": true, + "EnableTxnEvalTracer": false, + "EnableUsageLog": false, + "EnableVerbosedTransactionSyncLogging": false, + "EndpointAddress": "127.0.0.1:0", + "FallbackDNSResolverAddress": "", + "ForceFetchTransactions": false, + "ForceRelayMessages": false, + "GossipFanout": 4, + "HeartbeatUpdateInterval": 600, + "HotDataDir": "", + "IncomingConnectionsLimit": 2400, + "IncomingMessageFilterBucketCount": 5, + "IncomingMessageFilterBucketSize": 512, + "LedgerSynchronousMode": 2, + "LogArchiveDir": "", + "LogArchiveMaxAge": "", + "LogArchiveName": "node.archive.log", + "LogFileDir": "", + "LogSizeLimit": 1073741824, + "MaxAPIBoxPerApplication": 100000, + "MaxAPIResourcesPerAccount": 100000, + "MaxAcctLookback": 4, + "MaxCatchpointDownloadDuration": 43200000000000, + "MaxConnectionsPerIP": 15, + "MinCatchpointFileDownloadBytesPerSecond": 20480, + "NetAddress": "", + "NetworkMessageTraceServer": "", + "NetworkProtocolVersion": "", + "NodeExporterListenAddress": ":9100", + "NodeExporterPath": "./node_exporter", + "OptimizeAccountsDatabaseOnStartup": false, + "OutgoingMessageFilterBucketCount": 3, + "OutgoingMessageFilterBucketSize": 128, + "P2PPersistPeerID": false, + "P2PPrivateKeyLocation": "", + "ParticipationKeysRefreshInterval": 60000000000, + "PeerConnectionsUpdateInterval": 3600, + "PeerPingPeriodSeconds": 0, + "PriorityPeers": {}, + "ProposalAssemblyTime": 500000000, + "PublicAddress": "", + "ReconnectTime": 60000000000, + "RenaissanceCatchupLnProvenWeight": 0, + "RenaissanceCatchupProto": "", + "RenaissanceCatchupRound": 0, + "RenaissanceCatchupVotersCommitment": "", + "ReservedFDs": 256, + "RestConnectionsHardLimit": 2048, + "RestConnectionsSoftLimit": 1024, + "RestReadTimeoutSeconds": 15, + "RestWriteTimeoutSeconds": 120, + "RunHosted": false, + "StateproofCatchupDir": "", + "StateproofDir": "", + "StorageEngine": "sqlite", + "SuggestedFeeBlockHistory": 3, + "SuggestedFeeSlidingWindowSize": 50, + "TLSCertFile": "", + "TLSKeyFile": "", + "TelemetryToLog": true, + "TrackerDBDir": "", + "TransactionSyncDataExchangeRate": 0, + "TransactionSyncSignificantMessageThreshold": 0, + "TxBacklogReservedCapacityPerPeer": 20, + "TxBacklogServiceRateWindowSeconds": 10, + "TxBacklogSize": 26000, + "TxIncomingFilterMaxSize": 500000, + "TxIncomingFilteringFlags": 1, + "TxPoolExponentialIncreaseFactor": 2, + "TxPoolSize": 75000, + "TxSyncIntervalSeconds": 60, + "TxSyncServeResponseSize": 1000000, + "TxSyncTimeoutSeconds": 30, + "UseXForwardedForAddressField": "", + "VerifiedTranscationsCacheSize": 150000 +}